Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift: handle interrupt_poll in inliner and emit stack map at faulting load PC (#14947)

Shoryamishra61cfb483240b4d
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs
index db50c52b55..1fcac42ebf 100644
--- a/cranelift/codegen/meta/src/shared/instructions.rs
+++ b/cranelift/codegen/meta/src/shared/instructions.rs
@@ -420,10 +420,6 @@ fn define_control_flow(
.operands_out(&[
Operand::new("next_load_ptr", iAddr).with_doc("memory location to load from next time")
])
- // As with `stack_switch`, this instruction is a call, in that "it
- // continues execution elsewhere". See reasoning at
- // https://github.com/bytecodealliance/wasmtime/pull/9078#issuecomment-2273869774.
- .call()
.can_load()
// It may transfer control to something that may store. Declaring this
// makes us a memory fence.
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rs
index b6f062bae0..75d77b2926 100644
--- a/cranelift/codegen/src/alias_analysis.rs
+++ b/cranelift/codegen/src/alias_analysis.rs
@@ -144,6 +144,7 @@ fn alias_regions_observed(
debug_assert_eq!(func.dfg.insts[inst].memflags_data(&func.dfg), memflags);
if opcode.is_return()
|| opcode.is_call()
+ || opcode == Opcode::InterruptPoll
|| opcode.can_trap()
// NB: the `opcode.can_trap()` check above only covers explicitly
// trapping instructions (like `trap` and `trapz`), not loads/stores
diff --git a/cranelift/codegen/src/inline.rs b/cranelift/codegen/src/inline.rs
index 712a49b91a..c29d08af63 100644
--- a/cranelift/codegen/src/inline.rs
+++ b/cranelift/codegen/src/inline.rs
@@ -493,7 +493,7 @@ fn inline_one(
allocs.set_inlined_value(callee, *callee_val, *inlined_val);
}
- if opcode.is_call() {
+ if opcode.is_safepoint() {
append_stack_map_entries(
func,
callee,
@@ -502,7 +502,9 @@ fn inline_one(
inlined_inst,
callee_inst,
);
+ }
+ if opcode.is_call() {
// When we are inlining a `try_call` call site, we need to merge
// the call site's exception table into the inlined calls'
// exception tables. This can involve rewriting regular `call`s
diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs
index 81acba9e96..60b0d00b5e 100644
--- a/cranelift/codegen/src/ir/instructions.rs
+++ b/cranelift/codegen/src/ir/instructions.rs
@@ -314,10 +314,10 @@ impl Opcode {
/// Is this instruction a GC safepoint?
///
- /// Safepoints are all kinds of calls, except for tail calls.
+ /// Safepoints are all kinds of non-tail calls, as well as `interrupt_poll`.
#[inline]
pub fn is_safepoint(self) -> bool {
- self.is_call() && !self.is_return()
+ (self.is_call() && !self.is_return()) || self == Opcode::InterruptPoll
}
}
@@ -643,10 +643,6 @@ impl InstructionData {
Self::Ternary {
opcode: Opcode::StackSwitch,
..
- }
- | Self::IntAddTrap {
- opcode: Opcode::InterruptPoll,
- ..
} => {
// These instructions aren't actually calls, but they have the
// .call() side effect, as they continue execution elsewhere.
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 82ba06f524..35adc732d7 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -4127,11 +4127,19 @@ impl MachInstEmit for Inst {
trap_code,
..
} => {
+ let stack_map = state.take_stack_map();
+
// Record the address of the load in the trap table so a signal
// handler can later distinguish whether a segfault is its
// fault.
sink.add_trap(trap_code);
+ // SIGSEGV reports the PC of the faulting load, so the stack map
+ // is associated with the load itself.
+ if let Some(stack_map) = stack_map {
+ sink.push_user_stack_map(state, sink.cur_offset(), stack_map);
+ }
+
// Emit `ldr dst, [load_ptr]`. Reuse the `dst` address as the
// destination of the dead load, since we are clobbering it
// anyway.
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 3390da875c..e5e8bdbcfb 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -1257,7 +1257,7 @@ impl MachInst for Inst {
fn is_safepoint(&self) -> bool {
match self {
- Inst::Call { .. } | Inst::CallInd { .. } => true,
+ Inst::Call { .. } | Inst::CallInd { .. } | Inst::InterruptPoll { .. } => true,
_ => false,
}
}
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 042b022bc6..053cedad85 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -656,10 +656,18 @@ pub(crate) fn emit(
trap_code,
..
} => {
+ let stack_map = state.take_stack_map();
+
// Record the address of load in the trap table so a signal handler
// can later distinguish whether a segfault is its fault.
sink.add_trap(*trap_code);
+ // SIGSEGV reports the PC of the faulting load, so the stack map
+ // is associated with the load itself.
+ if let Some(stack_map) = stack_map {
+ sink.push_user_stack_map(state, sink.cur_offset(), stack_map);
+ }
+
let load_ptr_addr = SyntheticAmode::real(Amode::imm_reg(0, **load_ptr));
// Since we're clobbering dst anyway to store the resume address,
// also use it as a destination for the dead load rather than
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 916e6954f5..4019858c46 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1519,9 +1519,10 @@ impl MachInst for Inst {
fn is_safepoint(&self) -> bool {
match self {
- Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::StackSwitchBasic { .. } => {
- true
- }
+ Inst::CallKnown { .. }
+ | Inst::CallUnknown { .. }
+ | Inst::StackSwitchBasic { .. }
+ | Inst::InterruptPoll { .. } => true,
_ => false,
}
}
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index bb9f957a26..440624f6cd 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -1848,33 +1848,34 @@ impl<I: VCodeInst> MachBuffer<I> {
/// Push a user stack map onto this buffer.
///
- /// The stack map is associated with the given `return_addr` code
- /// offset. This must be the PC for the instruction just *after* this stack
- /// map's associated instruction. For example in the sequence `call $foo;
- /// add r8, rax`, the `return_addr` must be the offset of the start of the
- /// `add` instruction.
+ /// The stack map is associated with the given `pc` code offset. For a
+ /// call, this is the return address: the PC of the instruction just
+ /// *after* the call. For a resumable fault (such as `interrupt_poll`),
+ /// this is the PC of the faulting instruction itself. For a stack switch,
+ /// this is the PC at which execution resumes after the switch. In all
+ /// cases, `pc` identifies the machine state described by the map.
///
- /// Stack maps must be pushed in sorted `return_addr` order.
+ /// Stack maps must be pushed in strictly increasing `pc` order.
pub fn push_user_stack_map(
&mut self,
emit_state: &I::State,
- return_addr: CodeOffset,
+ pc: CodeOffset,
mut stack_map: ir::UserStackMap,
) {
let span = emit_state.frame_layout().active_size();
- trace!("Adding user stack map @ {return_addr:#x} spanning {span} bytes: {stack_map:?}");
+ trace!("Adding user stack map @ {pc:#x} spanning {span} bytes: {stack_map:?}");
debug_assert!(
self.user_stack_maps
.last()
- .map_or(true, |(prev_addr, _, _)| *prev_addr < return_addr),
+ .map_or(true, |(prev_addr, _, _)| *prev_addr < pc),
"pushed stack maps out of order: {} is not less than {}",
self.user_stack_maps.last().unwrap().0,
- return_addr,
+ pc,
);
stack_map.finalize(emit_state.frame_layout().sp_to_sized_stack_slots());
- self.user_stack_maps.push((return_addr, span, stack_map));
+ self.user_stack_maps.push((pc, span, stack_map));
}
/// Push a user stack map whose offsets are already relative to the stack
diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs
index 6dbe790d37..28c987db52 100644
--- a/cranelift/codegen/src/write.rs
+++ b/cranelift/codegen/src/write.rs
@@ -463,8 +463,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
Call {
func_ref, ref args, ..
} => {
- write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool)))?;
- write_user_stack_map_entries(w, dfg, inst)
+ write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool)))
}
CallIndirect {
sig_ref, ref args, ..
@@ -476,8 +475,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
sig_ref,
args[0],
DisplayValues(&args[1..])
- )?;
- write_user_stack_map_entries(w, dfg, inst)
+ )
}
TryCall {
func_ref,
@@ -491,8 +489,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
func_ref,
DisplayValues(args.as_slice(pool)),
exception_tables[exception].display(pool),
- )?;
- write_user_stack_map_entries(w, dfg, inst)
+ )
}
TryCallIndirect {
ref args,
@@ -506,8 +503,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
args[0],
DisplayValues(&args[1..]),
exception_tables[exception].display(pool),
- )?;
- write_user_stack_map_entries(w, dfg, inst)
+ )
}
FuncAddr { func_ref, .. } => write!(w, " {func_ref}"),
StackAddr {
@@ -534,6 +530,12 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
ExceptionHandlerAddress { block, imm, .. } => write!(w, " {block}, {imm}"),
}?;
+ // User stack maps are only valid on safepoints, but safepoints are not
+ // limited to the call instruction formats: `interrupt_poll` and
+ // `stack_switch` are also safepoints. Print maps for any instruction that
+ // has them rather than only for the call formats.
+ write_user_stack_map_entries(w, dfg, inst)?;
+
let mut sep = " ; ";
for arg in dfg.inst_values(inst) {
if let ValueDef::Result(src, _) = dfg.value_def(arg) {
diff --git a/cranelift/filetests/filetests/inline/interrupt-poll.clif b/cranelift/filetests/filetests/inline/interrupt-poll.clif
new file mode 100644
index 0000000000..a7c755a737
--- /dev/null
+++ b/cranelift/filetests/filetests/inline/interrupt-poll.clif
@@ -0,0 +1,124 @@
+test inline precise-output
+target x86_64
+
+;; Verify that interrupt_poll is preserved during inlining and that
+;; its user stack maps are copied and remapped into the caller.
+
+;; A callee with a poll and its own stack map, inlined at a normal call site
+;; that also has a stack map. The two maps should be merged.
+function %poll_callee(i64, i64, i64) -> i64 tail {
+ ss0 = explicit_slot 8, align = 8
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = stack_addr.i64 ss0
+ store notrap v0, v3
+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]
+ return v4
+}
+
+; (no functions inlined into %poll_callee)
+
+function %poll_caller(i64, i64, i64) -> i64 tail {
+ ss0 = explicit_slot 16, align = 8
+ fn0 = %poll_callee(i64, i64, i64) -> i64 tail
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = stack_addr.i64 ss0+8
+ store notrap v0, v3
+ v4 = call fn0(v0, v1, v2), stack_map=[i64 @ ss0+8]
+ v5 = stack_addr.i64 ss0+8
+ v6 = load.i64 notrap v5
+ v7 = iadd v4, v6
+ return v7
+}
+
+; function %poll_caller(i64, i64, i64) -> i64 tail {
+; ss0 = explicit_slot 16, align = 8
+; ss1 = explicit_slot 8, align = 8
+; sig0 = (i64, i64, i64) -> i64 tail
+; fn0 = %poll_callee sig0
+;
+; block0(v0: i64, v1: i64, v2: i64):
+; v3 = stack_addr.i64 ss0+8
+; store notrap v0, v3
+; jump block1
+;
+; block1:
+; v9 = stack_addr.i64 ss1
+; store.i64 notrap v0, v9
+; v10 = interrupt_poll.i64 v1, v2, user3, stack_map=[i64 @ ss0+8, i64 @ ss1+0]
+; jump block2(v10)
+;
+; block2(v8: i64):
+; v4 -> v8
+; v5 = stack_addr.i64 ss0+8
+; v6 = load.i64 notrap v5
+; v7 = iadd v4, v6
+; return v7
+; }
+
+;; A callee with both a poll and a real call, inlined at a `try_call` site: the
+;; real call has to be rewritten to a `try_call`, but the poll must be left
+;; alone.
+function %poll_callee_try(i64, i64, i64) -> i64 tail {
+ ss0 = explicit_slot 8, align = 8
+ sig0 = (i64) -> i64 tail
+ fn0 = %foo sig0
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = stack_addr.i64 ss0
+ store notrap v0, v3
+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]
+ v5 = call fn0(v4)
+ return v5
+}
+
+; (no functions inlined into %poll_callee_try)
+
+function %poll_try_caller(i64, i64, i64) -> i64 tail {
+ ss0 = explicit_slot 16, align = 8
+ sig0 = (i64, i64, i64) -> i64 tail
+ fn0 = %poll_callee_try(i64, i64, i64) -> i64 tail
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = stack_addr.i64 ss0+8
+ store notrap v0, v3
+ try_call fn0(v0, v1, v2), sig0, block1(ret0), [default: block2(exn0)], stack_map=[i64 @ ss0+8]
+block1(v4: i64):
+ v5 = stack_addr.i64 ss0+8
+ v6 = load.i64 notrap v5
+ v7 = iadd v4, v6
+ return v7
+block2(v8: i64):
+ return v8
+}
+
+; function %poll_try_caller(i64, i64, i64) -> i64 tail {
+; ss0 = explicit_slot 16, align = 8
+; ss1 = explicit_slot 8, align = 8
+; sig0 = (i64, i64, i64) -> i64 tail
+; sig1 = (i64, i64, i64) -> i64 tail
+; sig2 = (i64) -> i64 tail
+; fn0 = %poll_callee_try sig1
+; fn1 = %foo sig2
+;
+; block0(v0: i64, v1: i64, v2: i64):
+; v3 = stack_addr.i64 ss0+8
+; store notrap v0, v3
+; jump block3
+;
+; block3:
+; v9 = stack_addr.i64 ss1
+; store.i64 notrap v0, v9
+; v10 = interrupt_poll.i64 v1, v2, user3, stack_map=[i64 @ ss0+8, i64 @ ss1+0]
+; try_call fn1(v10), sig2, block4(ret0), [ default: block2(exn0) ], stack_map=[i64 @ ss0+8]
+;
+; block4(v12: i64):
+; v11 -> v12
+; jump block1(v11)
+;
+; block1(v4: i64):
+; v5 = stack_addr.i64 ss0+8
+; v6 = load.i64 notrap v5
+; v7 = iadd v4, v6
+; return v7
+;
+; block2(v8: i64):
+; return v8
+; }
diff --git a/cranelift/filetests/filetests/isa/aarch64/interrupt-poll-stack-map.clif b/cranelift/filetests/filetests/isa/aarch64/interrupt-poll-stack-map.clif
new file mode 100644
index 0000000000..d50a05ee97
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/interrupt-poll-stack-map.clif
@@ -0,0 +1,44 @@
+test compile precise-output
+target aarch64
+
+;; The stack map attached to an `interrupt_poll` must be emitted at the address
+;; of the faulting load, because the MMU-interruption handler resumes by
+;; re-executing that load and records its address as the Wasm resume PC.
+function %interrupt_poll_stack_map(i64, i64, i64) -> i64 tail {
+ ss0 = explicit_slot 8, align = 8
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = stack_addr.i64 ss0
+ store notrap v0, v3
+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]
+ return v4
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; sub sp, sp, #16
+; block0:
+; str x2, [sp]
+; mov x0, x4
+; mov x10, x3
+; x10 = interrupt_poll load_ptr=x10, context=x0, scratch=x9 #trap=user3
+; ; UserStackMap { by_type: [(types::I64, CompoundBitSet {0})], sp_to_sized_stack_slots: None }
+; mov x2, x10
+; add sp, sp, #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; sub sp, sp, #0x10
+; block1: ; offset 0xc
+; stur x2, [sp]
+; mov x0, x4
+; mov x10, x3
+; ldr x9, [x10] ; trap: user3
+; mov x2, x10
+; add sp, sp, #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
diff --git a/cranelift/filetests/filetests/isa/x64/interrupt-poll-stack-map.clif b/cranelift/filetests/filetests/isa/x64/interrupt-poll-stack-map.clif
new file mode 100644
index 0000000000..d169bac988
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/interrupt-poll-stack-map.clif
@@ -0,0 +1,46 @@
+test compile precise-output
+target x86_64
+
+;; The stack map attached to an `interrupt_poll` must be emitted at the address
+;; of the faulting load, because the MMU-interruption handler resumes by
+;; re-executing that load and records its address as the Wasm resume PC.
+function %interrupt_poll_stack_map(i64, i64, i64) -> i64 tail {
+ ss0 = explicit_slot 8, align = 8
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = stack_addr.i64 ss0
+ store notrap v0, v3
+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]
+ return v4
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x10, %rsp
+; block0:
+; movq %rdi, <offset:1>+(%rsp)
+; movq %rdx, %rdi
+; movq %rsi, %r11
+; %r11 = interrupt_poll load_ptr=%r11, context=%rdi, scratch=%r10 #trap=user3
+; ; UserStackMap { by_type: [(types::I64, CompoundBitSet {0})], sp_to_sized_stack_slots: None }
+; movq %r11, %rax
+; addq $0x10, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x10, %rsp
+; block1: ; offset 0x8
+; movq %rdi, (%rsp)
+; movq %rdx, %rdi
+; movq %rsi, %r11
+; movq (%r11), %r10 ; trap: user3
+; movq %r11, %rax
+; addq $0x10, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
diff --git a/cranelift/filetests/filetests/verifier/interrupt-poll.clif b/cranelift/filetests/filetests/verifier/interrupt-poll.clif
index 406d5c03d7..698dfc43c3 100644
--- a/cranelift/filetests/filetests/verifier/interrupt-poll.clif
+++ b/cranelift/filetests/filetests/verifier/interrupt-poll.clif
@@ -11,3 +11,9 @@ block0(v0: i64, v1: i64):
v2 = interrupt_poll v0, v1, user3 ; error: `interrupt_poll` requires the `tail` calling convention
return v2
}
+
+function %debug_tags(i64, i64) -> i64 tail {
+block0(v0: i64, v1: i64):
+<0, 1> v2 = interrupt_poll v0, v1, user3 ; error: debug tags present on non-call, non-sequence point instruction
+ return v2
+}
diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs
index f62b61043b..b2fcfb6996 100644
--- a/cranelift/reader/src/parser.rs
+++ b/cranelift/reader/src/parser.rs
@@ -2282,10 +2282,10 @@ impl<'a> Parser<'a> {
Token::Identifier("stack_map"),
"expected `stack_map = [...]`",
)?;
- if !opcode.is_call() || opcode.is_return() {
+ if !opcode.is_safepoint() {
return err!(
self.loc,
- "stack map can only be attached to a (non-tail) call"
+ "stack map can only be attached to a safepoint (non-tail call or interrupt_poll)"
);
}