A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift: handle interrupt_poll in inliner and emit stack map at faulting load PC (#14947)
Shoryamishra61cfb483240b4d
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rsindex db50c52b55..1fcac42ebf 100644--- a/cranelift/codegen/meta/src/shared/instructions.rs+++ b/cranelift/codegen/meta/src/shared/instructions.rs@@ -420,10 +420,6 @@ fn define_control_flow(.operands_out(&[Operand::new("next_load_ptr", iAddr).with_doc("memory location to load from next time")])- // As with `stack_switch`, this instruction is a call, in that "it- // continues execution elsewhere". See reasoning at- // https://github.com/bytecodealliance/wasmtime/pull/9078#issuecomment-2273869774.- .call().can_load()// It may transfer control to something that may store. Declaring this// makes us a memory fence.diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rsindex b6f062bae0..75d77b2926 100644--- a/cranelift/codegen/src/alias_analysis.rs+++ b/cranelift/codegen/src/alias_analysis.rs@@ -144,6 +144,7 @@ fn alias_regions_observed(debug_assert_eq!(func.dfg.insts[inst].memflags_data(&func.dfg), memflags);if opcode.is_return()|| opcode.is_call()+ || opcode == Opcode::InterruptPoll|| opcode.can_trap()// NB: the `opcode.can_trap()` check above only covers explicitly// trapping instructions (like `trap` and `trapz`), not loads/storesdiff --git a/cranelift/codegen/src/inline.rs b/cranelift/codegen/src/inline.rsindex 712a49b91a..c29d08af63 100644--- a/cranelift/codegen/src/inline.rs+++ b/cranelift/codegen/src/inline.rs@@ -493,7 +493,7 @@ fn inline_one(allocs.set_inlined_value(callee, *callee_val, *inlined_val);}- if opcode.is_call() {+ if opcode.is_safepoint() {append_stack_map_entries(func,callee,@@ -502,7 +502,9 @@ fn inline_one(inlined_inst,callee_inst,);+ }+ if opcode.is_call() {// When we are inlining a `try_call` call site, we need to merge// the call site's exception table into the inlined calls'// exception tables. This can involve rewriting regular `call`sdiff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rsindex 81acba9e96..60b0d00b5e 100644--- a/cranelift/codegen/src/ir/instructions.rs+++ b/cranelift/codegen/src/ir/instructions.rs@@ -314,10 +314,10 @@ impl Opcode {/// Is this instruction a GC safepoint?///- /// Safepoints are all kinds of calls, except for tail calls.+ /// Safepoints are all kinds of non-tail calls, as well as `interrupt_poll`.#[inline]pub fn is_safepoint(self) -> bool {- self.is_call() && !self.is_return()+ (self.is_call() && !self.is_return()) || self == Opcode::InterruptPoll}}@@ -643,10 +643,6 @@ impl InstructionData {Self::Ternary {opcode: Opcode::StackSwitch,..- }- | Self::IntAddTrap {- opcode: Opcode::InterruptPoll,- ..} => {// These instructions aren't actually calls, but they have the// .call() side effect, as they continue execution elsewhere.diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex 82ba06f524..35adc732d7 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -4127,11 +4127,19 @@ impl MachInstEmit for Inst {trap_code,..} => {+ let stack_map = state.take_stack_map();+// Record the address of the load in the trap table so a signal// handler can later distinguish whether a segfault is its// fault.sink.add_trap(trap_code);+ // SIGSEGV reports the PC of the faulting load, so the stack map+ // is associated with the load itself.+ if let Some(stack_map) = stack_map {+ sink.push_user_stack_map(state, sink.cur_offset(), stack_map);+ }+// Emit `ldr dst, [load_ptr]`. Reuse the `dst` address as the// destination of the dead load, since we are clobbering it// anyway.diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex 3390da875c..e5e8bdbcfb 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -1257,7 +1257,7 @@ impl MachInst for Inst {fn is_safepoint(&self) -> bool {match self {- Inst::Call { .. } | Inst::CallInd { .. } => true,+ Inst::Call { .. } | Inst::CallInd { .. } | Inst::InterruptPoll { .. } => true,_ => false,}}diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rsindex 042b022bc6..053cedad85 100644--- a/cranelift/codegen/src/isa/x64/inst/emit.rs+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs@@ -656,10 +656,18 @@ pub(crate) fn emit(trap_code,..} => {+ let stack_map = state.take_stack_map();+// Record the address of load in the trap table so a signal handler// can later distinguish whether a segfault is its fault.sink.add_trap(*trap_code);+ // SIGSEGV reports the PC of the faulting load, so the stack map+ // is associated with the load itself.+ if let Some(stack_map) = stack_map {+ sink.push_user_stack_map(state, sink.cur_offset(), stack_map);+ }+let load_ptr_addr = SyntheticAmode::real(Amode::imm_reg(0, **load_ptr));// Since we're clobbering dst anyway to store the resume address,// also use it as a destination for the dead load rather thandiff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex 916e6954f5..4019858c46 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -1519,9 +1519,10 @@ impl MachInst for Inst {fn is_safepoint(&self) -> bool {match self {- Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::StackSwitchBasic { .. } => {- true- }+ Inst::CallKnown { .. }+ | Inst::CallUnknown { .. }+ | Inst::StackSwitchBasic { .. }+ | Inst::InterruptPoll { .. } => true,_ => false,}}diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rsindex bb9f957a26..440624f6cd 100644--- a/cranelift/codegen/src/machinst/buffer.rs+++ b/cranelift/codegen/src/machinst/buffer.rs@@ -1848,33 +1848,34 @@ impl<I: VCodeInst> MachBuffer<I> {/// Push a user stack map onto this buffer.///- /// The stack map is associated with the given `return_addr` code- /// offset. This must be the PC for the instruction just *after* this stack- /// map's associated instruction. For example in the sequence `call $foo;- /// add r8, rax`, the `return_addr` must be the offset of the start of the- /// `add` instruction.+ /// The stack map is associated with the given `pc` code offset. For a+ /// call, this is the return address: the PC of the instruction just+ /// *after* the call. For a resumable fault (such as `interrupt_poll`),+ /// this is the PC of the faulting instruction itself. For a stack switch,+ /// this is the PC at which execution resumes after the switch. In all+ /// cases, `pc` identifies the machine state described by the map.///- /// Stack maps must be pushed in sorted `return_addr` order.+ /// Stack maps must be pushed in strictly increasing `pc` order.pub fn push_user_stack_map(&mut self,emit_state: &I::State,- return_addr: CodeOffset,+ pc: CodeOffset,mut stack_map: ir::UserStackMap,) {let span = emit_state.frame_layout().active_size();- trace!("Adding user stack map @ {return_addr:#x} spanning {span} bytes: {stack_map:?}");+ trace!("Adding user stack map @ {pc:#x} spanning {span} bytes: {stack_map:?}");debug_assert!(self.user_stack_maps.last()- .map_or(true, |(prev_addr, _, _)| *prev_addr < return_addr),+ .map_or(true, |(prev_addr, _, _)| *prev_addr < pc),"pushed stack maps out of order: {} is not less than {}",self.user_stack_maps.last().unwrap().0,- return_addr,+ pc,);stack_map.finalize(emit_state.frame_layout().sp_to_sized_stack_slots());- self.user_stack_maps.push((return_addr, span, stack_map));+ self.user_stack_maps.push((pc, span, stack_map));}/// Push a user stack map whose offsets are already relative to the stackdiff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rsindex 6dbe790d37..28c987db52 100644--- a/cranelift/codegen/src/write.rs+++ b/cranelift/codegen/src/write.rs@@ -463,8 +463,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmtCall {func_ref, ref args, ..} => {- write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool)))?;- write_user_stack_map_entries(w, dfg, inst)+ write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool)))}CallIndirect {sig_ref, ref args, ..@@ -476,8 +475,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmtsig_ref,args[0],DisplayValues(&args[1..])- )?;- write_user_stack_map_entries(w, dfg, inst)+ )}TryCall {func_ref,@@ -491,8 +489,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmtfunc_ref,DisplayValues(args.as_slice(pool)),exception_tables[exception].display(pool),- )?;- write_user_stack_map_entries(w, dfg, inst)+ )}TryCallIndirect {ref args,@@ -506,8 +503,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmtargs[0],DisplayValues(&args[1..]),exception_tables[exception].display(pool),- )?;- write_user_stack_map_entries(w, dfg, inst)+ )}FuncAddr { func_ref, .. } => write!(w, " {func_ref}"),StackAddr {@@ -534,6 +530,12 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmtExceptionHandlerAddress { block, imm, .. } => write!(w, " {block}, {imm}"),}?;+ // User stack maps are only valid on safepoints, but safepoints are not+ // limited to the call instruction formats: `interrupt_poll` and+ // `stack_switch` are also safepoints. Print maps for any instruction that+ // has them rather than only for the call formats.+ write_user_stack_map_entries(w, dfg, inst)?;+let mut sep = " ; ";for arg in dfg.inst_values(inst) {if let ValueDef::Result(src, _) = dfg.value_def(arg) {diff --git a/cranelift/filetests/filetests/inline/interrupt-poll.clif b/cranelift/filetests/filetests/inline/interrupt-poll.clifnew file mode 100644index 0000000000..a7c755a737--- /dev/null+++ b/cranelift/filetests/filetests/inline/interrupt-poll.clif@@ -0,0 +1,124 @@+test inline precise-output+target x86_64++;; Verify that interrupt_poll is preserved during inlining and that+;; its user stack maps are copied and remapped into the caller.++;; A callee with a poll and its own stack map, inlined at a normal call site+;; that also has a stack map. The two maps should be merged.+function %poll_callee(i64, i64, i64) -> i64 tail {+ ss0 = explicit_slot 8, align = 8+block0(v0: i64, v1: i64, v2: i64):+ v3 = stack_addr.i64 ss0+ store notrap v0, v3+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]+ return v4+}++; (no functions inlined into %poll_callee)++function %poll_caller(i64, i64, i64) -> i64 tail {+ ss0 = explicit_slot 16, align = 8+ fn0 = %poll_callee(i64, i64, i64) -> i64 tail+block0(v0: i64, v1: i64, v2: i64):+ v3 = stack_addr.i64 ss0+8+ store notrap v0, v3+ v4 = call fn0(v0, v1, v2), stack_map=[i64 @ ss0+8]+ v5 = stack_addr.i64 ss0+8+ v6 = load.i64 notrap v5+ v7 = iadd v4, v6+ return v7+}++; function %poll_caller(i64, i64, i64) -> i64 tail {+; ss0 = explicit_slot 16, align = 8+; ss1 = explicit_slot 8, align = 8+; sig0 = (i64, i64, i64) -> i64 tail+; fn0 = %poll_callee sig0+;+; block0(v0: i64, v1: i64, v2: i64):+; v3 = stack_addr.i64 ss0+8+; store notrap v0, v3+; jump block1+;+; block1:+; v9 = stack_addr.i64 ss1+; store.i64 notrap v0, v9+; v10 = interrupt_poll.i64 v1, v2, user3, stack_map=[i64 @ ss0+8, i64 @ ss1+0]+; jump block2(v10)+;+; block2(v8: i64):+; v4 -> v8+; v5 = stack_addr.i64 ss0+8+; v6 = load.i64 notrap v5+; v7 = iadd v4, v6+; return v7+; }++;; A callee with both a poll and a real call, inlined at a `try_call` site: the+;; real call has to be rewritten to a `try_call`, but the poll must be left+;; alone.+function %poll_callee_try(i64, i64, i64) -> i64 tail {+ ss0 = explicit_slot 8, align = 8+ sig0 = (i64) -> i64 tail+ fn0 = %foo sig0+block0(v0: i64, v1: i64, v2: i64):+ v3 = stack_addr.i64 ss0+ store notrap v0, v3+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]+ v5 = call fn0(v4)+ return v5+}++; (no functions inlined into %poll_callee_try)++function %poll_try_caller(i64, i64, i64) -> i64 tail {+ ss0 = explicit_slot 16, align = 8+ sig0 = (i64, i64, i64) -> i64 tail+ fn0 = %poll_callee_try(i64, i64, i64) -> i64 tail+block0(v0: i64, v1: i64, v2: i64):+ v3 = stack_addr.i64 ss0+8+ store notrap v0, v3+ try_call fn0(v0, v1, v2), sig0, block1(ret0), [default: block2(exn0)], stack_map=[i64 @ ss0+8]+block1(v4: i64):+ v5 = stack_addr.i64 ss0+8+ v6 = load.i64 notrap v5+ v7 = iadd v4, v6+ return v7+block2(v8: i64):+ return v8+}++; function %poll_try_caller(i64, i64, i64) -> i64 tail {+; ss0 = explicit_slot 16, align = 8+; ss1 = explicit_slot 8, align = 8+; sig0 = (i64, i64, i64) -> i64 tail+; sig1 = (i64, i64, i64) -> i64 tail+; sig2 = (i64) -> i64 tail+; fn0 = %poll_callee_try sig1+; fn1 = %foo sig2+;+; block0(v0: i64, v1: i64, v2: i64):+; v3 = stack_addr.i64 ss0+8+; store notrap v0, v3+; jump block3+;+; block3:+; v9 = stack_addr.i64 ss1+; store.i64 notrap v0, v9+; v10 = interrupt_poll.i64 v1, v2, user3, stack_map=[i64 @ ss0+8, i64 @ ss1+0]+; try_call fn1(v10), sig2, block4(ret0), [ default: block2(exn0) ], stack_map=[i64 @ ss0+8]+;+; block4(v12: i64):+; v11 -> v12+; jump block1(v11)+;+; block1(v4: i64):+; v5 = stack_addr.i64 ss0+8+; v6 = load.i64 notrap v5+; v7 = iadd v4, v6+; return v7+;+; block2(v8: i64):+; return v8+; }diff --git a/cranelift/filetests/filetests/isa/aarch64/interrupt-poll-stack-map.clif b/cranelift/filetests/filetests/isa/aarch64/interrupt-poll-stack-map.clifnew file mode 100644index 0000000000..d50a05ee97--- /dev/null+++ b/cranelift/filetests/filetests/isa/aarch64/interrupt-poll-stack-map.clif@@ -0,0 +1,44 @@+test compile precise-output+target aarch64++;; The stack map attached to an `interrupt_poll` must be emitted at the address+;; of the faulting load, because the MMU-interruption handler resumes by+;; re-executing that load and records its address as the Wasm resume PC.+function %interrupt_poll_stack_map(i64, i64, i64) -> i64 tail {+ ss0 = explicit_slot 8, align = 8+block0(v0: i64, v1: i64, v2: i64):+ v3 = stack_addr.i64 ss0+ store notrap v0, v3+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]+ return v4+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; sub sp, sp, #16+; block0:+; str x2, [sp]+; mov x0, x4+; mov x10, x3+; x10 = interrupt_poll load_ptr=x10, context=x0, scratch=x9 #trap=user3+; ; UserStackMap { by_type: [(types::I64, CompoundBitSet {0})], sp_to_sized_stack_slots: None }+; mov x2, x10+; add sp, sp, #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; sub sp, sp, #0x10+; block1: ; offset 0xc+; stur x2, [sp]+; mov x0, x4+; mov x10, x3+; ldr x9, [x10] ; trap: user3+; mov x2, x10+; add sp, sp, #0x10+; ldp x29, x30, [sp], #0x10+; retdiff --git a/cranelift/filetests/filetests/isa/x64/interrupt-poll-stack-map.clif b/cranelift/filetests/filetests/isa/x64/interrupt-poll-stack-map.clifnew file mode 100644index 0000000000..d169bac988--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/interrupt-poll-stack-map.clif@@ -0,0 +1,46 @@+test compile precise-output+target x86_64++;; The stack map attached to an `interrupt_poll` must be emitted at the address+;; of the faulting load, because the MMU-interruption handler resumes by+;; re-executing that load and records its address as the Wasm resume PC.+function %interrupt_poll_stack_map(i64, i64, i64) -> i64 tail {+ ss0 = explicit_slot 8, align = 8+block0(v0: i64, v1: i64, v2: i64):+ v3 = stack_addr.i64 ss0+ store notrap v0, v3+ v4 = interrupt_poll v1, v2, user3, stack_map=[i64 @ ss0+0]+ return v4+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; subq $0x10, %rsp+; block0:+; movq %rdi, <offset:1>+(%rsp)+; movq %rdx, %rdi+; movq %rsi, %r11+; %r11 = interrupt_poll load_ptr=%r11, context=%rdi, scratch=%r10 #trap=user3+; ; UserStackMap { by_type: [(types::I64, CompoundBitSet {0})], sp_to_sized_stack_slots: None }+; movq %r11, %rax+; addq $0x10, %rsp+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; subq $0x10, %rsp+; block1: ; offset 0x8+; movq %rdi, (%rsp)+; movq %rdx, %rdi+; movq %rsi, %r11+; movq (%r11), %r10 ; trap: user3+; movq %r11, %rax+; addq $0x10, %rsp+; movq %rbp, %rsp+; popq %rbp+; retqdiff --git a/cranelift/filetests/filetests/verifier/interrupt-poll.clif b/cranelift/filetests/filetests/verifier/interrupt-poll.clifindex 406d5c03d7..698dfc43c3 100644--- a/cranelift/filetests/filetests/verifier/interrupt-poll.clif+++ b/cranelift/filetests/filetests/verifier/interrupt-poll.clif@@ -11,3 +11,9 @@ block0(v0: i64, v1: i64):v2 = interrupt_poll v0, v1, user3 ; error: `interrupt_poll` requires the `tail` calling conventionreturn v2}++function %debug_tags(i64, i64) -> i64 tail {+block0(v0: i64, v1: i64):+<0, 1> v2 = interrupt_poll v0, v1, user3 ; error: debug tags present on non-call, non-sequence point instruction+ return v2+}diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rsindex f62b61043b..b2fcfb6996 100644--- a/cranelift/reader/src/parser.rs+++ b/cranelift/reader/src/parser.rs@@ -2282,10 +2282,10 @@ impl<'a> Parser<'a> {Token::Identifier("stack_map"),"expected `stack_map = [...]`",)?;- if !opcode.is_call() || opcode.is_return() {+ if !opcode.is_safepoint() {return err!(self.loc,- "stack map can only be attached to a (non-tail) call"+ "stack map can only be attached to a safepoint (non-tail call or interrupt_poll)");}