Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: implicitly consider trapping blocks as "cold" during lowering (#13689)

Nick Fitzgeraldd95a3ec9e5a6
diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs
index ee1d2debae..a194246ad8 100644
--- a/cranelift/codegen/src/ir/function.rs
+++ b/cranelift/codegen/src/ir/function.rs
@@ -473,6 +473,20 @@ impl Function {
pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef {
self.stencil.dfg.ext_funcs.push(data)
}
+
+ /// Is the given block marked `cold` or otherwise effectively `cold` in
+ /// practice?
+ pub fn is_effectively_cold(&self, block: Block) -> bool {
+ if self.layout.is_cold(block) {
+ return true;
+ }
+
+ // Blocks that unconditionally trap are effectively
+ // also cold.
+ self.layout
+ .last_inst(block)
+ .is_some_and(|inst| self.dfg.insts[inst].opcode() == ir::Opcode::Trap)
+ }
}
/// Wrapper type capable of displaying a `Function`.
diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs
index 215b204eee..f19720e6c2 100644
--- a/cranelift/codegen/src/machinst/blockorder.rs
+++ b/cranelift/codegen/src/machinst/blockorder.rs
@@ -257,7 +257,7 @@ impl BlockLoweringOrder {
lowered_succ_indices
.extend(block_succs[range].iter().map(|lb| lb_to_bindex[lb]));
- if f.layout.is_cold(block) {
+ if f.is_effectively_cold(block) {
cold_blocks.insert(bindex);
}
@@ -282,7 +282,7 @@ impl BlockLoweringOrder {
// Edges inherit indirect branch and cold block metadata from their
// successor.
- if f.layout.is_cold(succ) {
+ if f.is_effectively_cold(succ) {
cold_blocks.insert(bindex);
}
diff --git a/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif b/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif
new file mode 100644
index 0000000000..a4d1863992
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif
@@ -0,0 +1,40 @@
+test compile precise-output
+target pulley64
+
+function %f(i8, i64) -> i64 {
+block0(v0: i8, v1: i64):
+ brif v0, block1, block2
+
+;; This block is implicitly cold because it ends in a trap; it should come last
+;; in the lowered block order.
+block1:
+ v2 = iconst.i32 42
+ store notrap aligned v2, v1
+ trap user123
+
+block2:
+ v3 = iconst.i64 0
+ return v3
+}
+
+; VCode:
+; block0:
+; zext8 x5, x0
+; br_if32 x5, label2; jump label1
+; block1:
+; xzero x0
+; ret
+; block2:
+; xconst8 x7, 42
+; xstore32 x1+0, x7 // flags = notrap aligned
+; trap // trap=TrapCode(123)
+;
+; Disassembled:
+; zext8 x5, x0
+; br_if32 x5, 0x9 // target = 0xc
+; xzero x0
+; ret
+; xconst8 x7, 42
+; xstore32le_o32 x1, 0, x7
+; trap
+
diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.wat
index c903d304a5..327084f2f5 100644
--- a/tests/disas/debug-exceptions.wat
+++ b/tests/disas/debug-exceptions.wat
@@ -31,35 +31,35 @@
;; stur x0, [sp, #0x18]
;; mov x0, sp
;; cmp x0, x1
-;; b.lo #0x21c
+;; b.lo #0x220
;; 48: stur x2, [sp]
;; mov x0, x2
;; stur x2, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x35, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [61, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [62, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [59, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [60, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3d, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [57, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [58, 1, 0, 148]
;; mov w1, #0x2a
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3f, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [54, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [55, 1, 0, 148]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x40, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [53, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [54, 1, 0, 148]
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [51, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [52, 1, 0, 148]
;; ldur x2, [sp, #0x10]
-;; bl #0x4b0
+;; bl #0x4b4
;; 88: ldur x0, [sp, #0x10]
;; mov x19, x2
;; ldr x1, [x0, #0x20]
@@ -68,7 +68,7 @@
;; add x2, x2, #0x20
;; ldr w4, [x1, #4]
;; cmp x2, x4
-;; b.hi #0x1b4
+;; b.hi #0x1a0
;; ac: ldur x0, [sp, #0x18]
;; add w4, w3, #0x20
;; str w4, [x1]
@@ -92,47 +92,41 @@
;; mov w11, #0
;; str w11, [x1, #0x14]
;; ldur x2, [sp, #0x10]
-;; bl #0x4e8
+;; bl #0x4ec
;; ├─╼ exception frame offset: SP = FP - 0xb0
-;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x124
-;; 10c: mov w3, #9
-;; ldur x2, [sp, #0x10]
-;; bl #0x444
-;; 118: ldur x2, [sp, #0x10]
-;; bl #0x47c
-;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; 120: udf #0xc11f
-;; mov w1, w0
+;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x110
+;; b #0x1d8
+;; 110: mov w1, w0
;; mov x2, #0x20
;; adds x15, x1, x2
;; cset x1, hs
;; tst w1, #0xff
-;; b.ne #0x204
-;; 13c: ldur x2, [sp, #0x18]
+;; b.ne #0x208
+;; 128: ldur x2, [sp, #0x18]
;; ldr x1, [x2, #0x28]
;; cmp x15, x1
-;; b.hi #0x1ec
-;; 14c: ldr x1, [x2, #0x20]
+;; b.hi #0x1f0
+;; 138: ldr x1, [x2, #0x20]
;; add x1, x1, #0x18
;; ldr w0, [x1, w0, uxtw]
;; stur w0, [sp, #8]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x48, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [250, 0, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [0, 1, 0, 148]
;; ldur x1, [sp, #0x10]
;; ldr x0, [x1, #0x38]
;; ldr x2, [x1, #0x48]
;; ldur x3, [sp, #0x10]
;; blr x0
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; 178: ldur x0, [sp, #0x10]
+;; 164: ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [243, 0, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [249, 0, 0, 148]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4b, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [242, 0, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [248, 0, 0, 148]
;; add sp, sp, #0x20
;; ldp d8, d9, [sp], #0x10
;; ldp d10, d11, [sp], #0x10
@@ -145,38 +139,45 @@
;; ldp x27, x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 1b4: mov w3, #2
-;; 1b8: movk w3, #0x400, lsl #16
-;; 1bc: ldur x0, [sp, #0x10]
-;; 1c0: ldr x4, [x0, #0x28]
-;; 1c4: ldr w4, [x4, #8]
-;; 1c8: mov w5, #0x20
-;; 1cc: mov w6, #0x10
-;; 1d0: ldur x2, [sp, #0x10]
-;; 1d4: bl #0x3dc
-;; 1d8: ldur x0, [sp, #0x18]
-;; 1dc: ldr x3, [x0, #0x20]
-;; 1e0: add x1, x3, w2, uxtw
-;; 1e4: mov x3, x2
-;; 1e8: b #0xec
-;; 1ec: mov w3, #0xfe
-;; 1f0: ldur x2, [sp, #0x10]
-;; 1f4: bl #0x444
-;; 1f8: ldur x2, [sp, #0x10]
-;; 1fc: bl #0x47c
+;; 1a0: mov w3, #2
+;; 1a4: movk w3, #0x400, lsl #16
+;; 1a8: ldur x0, [sp, #0x10]
+;; 1ac: ldr x4, [x0, #0x28]
+;; 1b0: ldr w4, [x4, #8]
+;; 1b4: mov w5, #0x20
+;; 1b8: mov w6, #0x10
+;; 1bc: ldur x2, [sp, #0x10]
+;; 1c0: bl #0x3e0
+;; 1c4: ldur x0, [sp, #0x18]
+;; 1c8: ldr x3, [x0, #0x20]
+;; 1cc: add x1, x3, w2, uxtw
+;; 1d0: mov x3, x2
+;; 1d4: b #0xec
+;; 1d8: mov w3, #9
+;; 1dc: ldur x2, [sp, #0x10]
+;; 1e0: bl #0x448
+;; 1e4: ldur x2, [sp, #0x10]
+;; 1e8: bl #0x480
+;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
+;; 1ec: udf #0xc11f
+;; 1f0: mov w3, #0xfe
+;; 1f4: ldur x2, [sp, #0x10]
+;; 1f8: bl #0x448
+;; 1fc: ldur x2, [sp, #0x10]
+;; 200: bl #0x480
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; 200: udf #0xc11f
-;; 204: mov w3, #0xfe
-;; 208: ldur x2, [sp, #0x10]
-;; 20c: bl #0x444
-;; 210: ldur x2, [sp, #0x10]
-;; 214: bl #0x47c
+;; 204: udf #0xc11f
+;; 208: mov w3, #0xfe
+;; 20c: ldur x2, [sp, #0x10]
+;; 210: bl #0x448
+;; 214: ldur x2, [sp, #0x10]
+;; 218: bl #0x480
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; 218: udf #0xc11f
-;; 21c: stur x2, [sp, #0x10]
-;; 220: mov w3, #0
-;; 224: bl #0x444
-;; 228: ldur x2, [sp, #0x10]
-;; 22c: bl #0x47c
+;; 21c: udf #0xc11f
+;; 220: stur x2, [sp, #0x10]
+;; 224: mov w3, #0
+;; 228: bl #0x448
+;; 22c: ldur x2, [sp, #0x10]
+;; 230: bl #0x480
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x34, slot at FP-0xb0, locals , stack
-;; 230: udf #0xc11f
+;; 234: udf #0xc11f
diff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.wat
index 8af24433e7..efee2b2931 100644
--- a/tests/disas/exceptions.wat
+++ b/tests/disas/exceptions.wat
@@ -23,7 +23,7 @@
;; movq 0x18(%r10), %r10
;; addq $0x50, %r10
;; cmpq %rsp, %r10
-;; ja 0xe7
+;; ja 0xec
;; 19: subq $0x40, %rsp
;; movq %rbx, 0x10(%rsp)
;; movq %r12, 0x18(%rsp)
@@ -41,7 +41,7 @@
;; leaq 0x30(%rcx), %rax
;; movl 4(%rdx), %edi
;; cmpq %rdi, %rax
-;; ja 0xb3
+;; ja 0xb6
;; 5f: leal 0x30(%rsi), %eax
;; movl %eax, (%rdx)
;; movq 8(%rbx), %rax
@@ -63,8 +63,8 @@
;; movq %rbx, %rdi
;; movq %rbx, (%rsp)
;; callq 0x418
-;; ud2
-;; movl $0x4000002, %esi
+;; jmp 0xea
+;; b6: movl $0x4000002, %esi
;; movq 0x28(%rbx), %rax
;; movl 0xc(%rax), %edx
;; movl $0x30, %ecx
@@ -77,7 +77,8 @@
;; movq %rax, %rsi
;; movq %r14, %rdi
;; jmp 0x8e
-;; e7: ud2
+;; ea: ud2
+;; ud2
;;
;; wasm[0]::function[1]::catch:
;; pushq %rbp