A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: add debug tag infrastructure. (#11768)
Chris Fallina3d6e407173d
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rsindex 7480a4d44e..9604c6904b 100644--- a/cranelift/codegen/meta/src/shared/instructions.rs+++ b/cranelift/codegen/meta/src/shared/instructions.rs@@ -3902,4 +3902,20 @@ pub(crate) fn define(Operand::new("a", &TxN.dynamic_to_vector()).with_doc("New fixed vector"),]),);++ ig.push(+ Inst::new(+ "sequence_point",+ r#"+ A compiler barrier that acts as an immovable marker from IR input to machine-code output.++ This "sequence point" can have debug tags attached to it, and these tags will be+ noted in the output `MachBuffer`.++ It prevents motion of any other side-effects across this boundary.+ "#,+ &formats.nullary,+ )+ .other_side_effects(),+ );}diff --git a/cranelift/codegen/src/inline.rs b/cranelift/codegen/src/inline.rsindex 9a425086fd..1ca82203f3 100644--- a/cranelift/codegen/src/inline.rs+++ b/cranelift/codegen/src/inline.rs@@ -20,7 +20,7 @@//! Cranelift the body of the callee that is to be inlined.use crate::cursor::{Cursor as _, FuncCursor};-use crate::ir::{self, ExceptionTableData, ExceptionTableItem, InstBuilder as _};+use crate::ir::{self, DebugTag, ExceptionTableData, ExceptionTableItem, InstBuilder as _};use crate::result::CodegenResult;use crate::trace;use crate::traversals::Dfs;@@ -366,6 +366,13 @@ fn inline_one(// callee.let mut last_inlined_block = inline_block_layout(func, call_block, callee, &entity_map);+ // Get a copy of debug tags on the call instruction; these are+ // prepended to debug tags on inlined instructions. Remove them+ // from the call itself as it will be rewritten to a jump (which+ // cannot have tags).+ let call_debug_tags = func.debug_tags.get(call_inst).to_vec();+ func.debug_tags.set(call_inst, []);+// Translate each instruction from the callee into the caller,// appending them to their associated block in the caller.//@@ -403,6 +410,29 @@ fn inline_one(let inlined_inst = func.dfg.make_inst(inlined_inst_data);func.layout.append_inst(inlined_inst, inlined_block);+ // Copy over debug tags, translating referenced entities+ // as appropriate.+ let debug_tags = callee.debug_tags.get(callee_inst);+ // If there are tags on the inlined instruction, we always+ // add tags, and we prepend any tags from the call+ // instruction; but we don't add tags if only the callsite+ // had them (this would otherwise mean that every single+ // instruction in an inlined function body would get+ // tags).+ if !debug_tags.is_empty() {+ let tags = call_debug_tags+ .iter()+ .cloned()+ .chain(debug_tags.iter().map(|tag| match *tag {+ DebugTag::User(value) => DebugTag::User(value),+ DebugTag::StackSlot(slot) => {+ DebugTag::StackSlot(entity_map.inlined_stack_slot(slot))+ }+ }))+ .collect::<SmallVec<[_; 4]>>();+ func.debug_tags.set(inlined_inst, tags);+ }+let opcode = callee.dfg.insts[callee_inst].opcode();if opcode.is_return() {// Instructions that return do not define any values, so wediff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rsindex cdaa5ebc25..7b42e17286 100644--- a/cranelift/codegen/src/inst_predicates.rs+++ b/cranelift/codegen/src/inst_predicates.rs@@ -147,7 +147,8 @@ pub fn has_memory_fence_semantics(op: Opcode) -> bool {| Opcode::AtomicLoad| Opcode::AtomicStore| Opcode::Fence- | Opcode::Debugtrap => true,+ | Opcode::Debugtrap+ | Opcode::SequencePoint => true,Opcode::Call | Opcode::CallIndirect | Opcode::TryCall | Opcode::TryCallIndirect => true,op if op.can_trap() => true,_ => false,diff --git a/cranelift/codegen/src/ir/debug_tags.rs b/cranelift/codegen/src/ir/debug_tags.rsnew file mode 100644index 0000000000..39d137c297--- /dev/null+++ b/cranelift/codegen/src/ir/debug_tags.rs@@ -0,0 +1,141 @@+//! Debug tag storage.+//!+//! Cranelift permits the embedder to place "debug tags" on+//! instructions in CLIF. These tags are sequences of items of various+//! kinds, with no other meaning imposed by Cranelift. They are passed+//! through to metadata provided alongside the compilation result.+//!+//! When Cranelift inlines a function, it will prepend any tags from+//! the call instruction at the inlining callsite to tags on all+//! inlined instructions.+//!+//! These tags can be used, for example, to identify stackslots that+//! store user state, or to denote positions in user source. In+//! general, the intent is to allow perfect reconstruction of original+//! (source-level) program state in an instrumentation-based+//! debug-info scheme, as long as the instruction(s) on which these+//! tags are attached are preserved. This will be the case for any+//! instructions with side-effects.+//!+//! A few answers to design questions that lead to this design:+//!+//! - Why not use the SourceLoc mechanism? Debug tags are richer than+//! that infrastructure because they preserve inlining location and+//! are interleaved properly with any other tags describing the+//! frame.+//! - Why not attach debug tags only to special sequence-point+//! instructions? This is driven by inlining: we should have the+//! semantic information about a callsite attached directly to the+//! call and observe it there, not have a magic "look backward to+//! find a sequence point" behavior in the inliner.+//!+//! In other words, the needs of preserving "virtual" frames across an+//! inlining transform drive this design.++use crate::ir::{Inst, StackSlot};+use alloc::collections::BTreeMap;+use alloc::vec::Vec;+use core::ops::Range;++/// Debug tags for instructions.+#[derive(Clone, PartialEq, Hash, Default)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub struct DebugTags {+ /// Pool of tags, referred to by `insts` below.+ tags: Vec<DebugTag>,++ /// Per-instruction range for its list of tags in the tag pool (if+ /// any).+ ///+ /// Note: we don't use `PackedOption` and `EntityList` here+ /// because the values that we are storing are not entities.+ insts: BTreeMap<Inst, Range<u32>>,+}++/// One debug tag.+#[derive(Clone, Debug, PartialEq, Hash)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub enum DebugTag {+ /// User-specified `u32` value, opaque to Cranelift.+ User(u32),++ /// A stack slot reference.+ StackSlot(StackSlot),+}++impl DebugTags {+ /// Set the tags on an instruction, overwriting existing tag list.+ ///+ /// Tags can only be set on call instructions (those for which+ /// [`crate::Opcode::is_call()`] returns `true`) and on+ /// `sequence_point` instructions. This property is checked by the+ /// CLIF verifier.+ pub fn set(&mut self, inst: Inst, tags: impl IntoIterator<Item = DebugTag>) {+ let start = u32::try_from(self.tags.len()).unwrap();+ self.tags.extend(tags);+ let end = u32::try_from(self.tags.len()).unwrap();+ if end > start {+ self.insts.insert(inst, start..end);+ } else {+ self.insts.remove(&inst);+ }+ }++ /// Get the tags associated with an instruction.+ pub fn get(&self, inst: Inst) -> &[DebugTag] {+ if let Some(range) = self.insts.get(&inst) {+ let start = usize::try_from(range.start).unwrap();+ let end = usize::try_from(range.end).unwrap();+ &self.tags[start..end]+ } else {+ &[]+ }+ }++ /// Does the given instruction have any tags?+ pub fn has(&self, inst: Inst) -> bool {+ // We rely on the invariant that an entry in the map is+ // present only if the list range is non-empty.+ self.insts.contains_key(&inst)+ }++ /// Clone the tags from one instruction to another.+ ///+ /// This clone is cheap (references the same underlying storage)+ /// because the tag lists are immutable.+ pub fn clone_tags(&mut self, from: Inst, to: Inst) {+ if let Some(range) = self.insts.get(&from).cloned() {+ self.insts.insert(to, range);+ } else {+ self.insts.remove(&to);+ }+ }++ /// Are any debug tags present?+ ///+ /// This is used for adjusting margins when pretty-printing CLIF.+ pub fn is_empty(&self) -> bool {+ self.insts.is_empty()+ }++ /// Clear all tags.+ pub fn clear(&mut self) {+ self.insts.clear();+ self.tags.clear();+ }+}++impl core::fmt::Display for DebugTag {+ fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {+ match self {+ DebugTag::User(value) => write!(f, "{value}"),+ DebugTag::StackSlot(slot) => write!(f, "{slot}"),+ }+ }+}diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rsindex 02f2a31cc7..80926edcdf 100644--- a/cranelift/codegen/src/ir/function.rs+++ b/cranelift/codegen/src/ir/function.rs@@ -5,6 +5,7 @@use crate::HashMap;use crate::entity::{PrimaryMap, SecondaryMap};+use crate::ir::DebugTags;use crate::ir::{self, Block, DataFlowGraph, DynamicStackSlot, DynamicStackSlotData, DynamicStackSlots,DynamicType, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Inst, JumpTable,@@ -190,6 +191,22 @@ pub struct FunctionStencil {/// interpreted by Cranelift, only preserved.pub srclocs: SourceLocs,+ /// Opaque debug-info tags on sequence-point and call+ /// instructions.+ ///+ /// These tags are not interpreted by Cranelift, and are passed+ /// through to compilation-result metadata. The only semantic+ /// structure that Cranelift imposes is that when inlining, it+ /// prepends the callsite call instruction's tags to the tags on+ /// inlined instructions.+ ///+ /// In order to ensure clarity around guaranteed compiler+ /// behavior, tags are only permitted on instructions whose+ /// presence and sequence will remain the same in the compiled+ /// output: namely, `sequence_point` instructions and ordinary+ /// call instructions.+ pub debug_tags: DebugTags,+/// An optional global value which represents an expression evaluating to/// the stack limit for this function. This `GlobalValue` will be/// interpreted in the prologue, if necessary, to insert a stack check to@@ -209,6 +226,7 @@ impl FunctionStencil {self.dfg.clear();self.layout.clear();self.srclocs.clear();+ self.debug_tags.clear();self.stack_limit = None;}@@ -408,6 +426,7 @@ impl Function {layout: Layout::new(),srclocs: SecondaryMap::new(),stack_limit: None,+ debug_tags: DebugTags::default(),},params: FunctionParameters::new(),}diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rsindex 4519d5d74e..eca17e7bad 100644--- a/cranelift/codegen/src/ir/mod.rs+++ b/cranelift/codegen/src/ir/mod.rs@@ -4,6 +4,7 @@ mod atomic_rmw_op;mod builder;pub mod condcodes;pub mod constant;+mod debug_tags;pub mod dfg;pub mod dynamic_type;pub mod entities;@@ -36,6 +37,7 @@ pub use crate::ir::builder::{InsertBuilder, InstBuilder, InstBuilderBase, InstInserterBase, ReplaceBuilder,};pub use crate::ir::constant::{ConstantData, ConstantPool};+pub use crate::ir::debug_tags::{DebugTag, DebugTags};pub use crate::ir::dfg::{BlockData, DataFlowGraph, ValueDef};pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes, dynamic_to_fixed};pub use crate::ir::entities::{@@ -64,7 +66,7 @@ pub use crate::ir::progpoint::ProgramPoint;pub use crate::ir::sourceloc::RelSourceLoc;pub use crate::ir::sourceloc::SourceLoc;pub use crate::ir::stackslot::{- DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKind, StackSlots,+ DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKey, StackSlotKind, StackSlots,};pub use crate::ir::trapcode::TrapCode;pub use crate::ir::types::Type;diff --git a/cranelift/codegen/src/ir/stackslot.rs b/cranelift/codegen/src/ir/stackslot.rsindex cfcb8b2c9c..1ad2a5d7d8 100644--- a/cranelift/codegen/src/ir/stackslot.rs+++ b/cranelift/codegen/src/ir/stackslot.rs@@ -69,6 +69,37 @@ pub struct StackSlotData {/// be aligned according to other considerations, such as minimum/// stack slot size or machine word size, as well.pub align_shift: u8,++ /// Opaque stackslot metadata handle, passed through to+ /// compilation result metadata describing stackslot location.+ ///+ /// In the face of compiler transforms like inlining that may move+ /// stackslots between functions, when an embedder wants to+ /// externally observe stackslots, it needs a first-class way for+ /// the identity of stackslots to be carried along with the IR+ /// entities. This opaque `StackSlotKey` allows the embedder to do+ /// so.+ pub key: Option<StackSlotKey>,+}++/// An opaque key uniquely identifying a stack slot.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]+pub struct StackSlotKey(u64);+impl StackSlotKey {+ /// Construct a [`StackSlotKey`] from raw bits.+ ///+ /// An embedder can use any 64-bit value to describe a stack slot;+ /// there are no restrictions, and the value does not mean+ /// anything to Cranelift itself.+ pub fn new(value: u64) -> StackSlotKey {+ StackSlotKey(value)+ }++ /// Get the raw bits from the [`StackSlotKey`].+ pub fn bits(&self) -> u64 {+ self.0+ }}impl StackSlotData {@@ -78,23 +109,40 @@ impl StackSlotData {kind,size,align_shift,+ key: None,+ }+ }++ /// Create a stack slot with the specified byte size and alignment+ /// and the given user-defined key.+ pub fn new_with_key(+ kind: StackSlotKind,+ size: StackSize,+ align_shift: u8,+ key: StackSlotKey,+ ) -> Self {+ Self {+ kind,+ size,+ align_shift,+ key: Some(key),}}}impl fmt::Display for StackSlotData {fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {- if self.align_shift != 0 {- write!(- f,- "{} {}, align = {}",- self.kind,- self.size,- 1u32 << self.align_shift- )+ let align_shift = if self.align_shift != 0 {+ format!(", align = {}", 1u32 << self.align_shift)} else {- write!(f, "{} {}", self.kind, self.size)- }+ "".into()+ };+ let key = match self.key {+ Some(value) => format!(", key = {}", value.bits()),+ None => "".into(),+ };++ write!(f, "{} {}{align_shift}{key}", self.kind, self.size)}}diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex 9251f4d8b1..2d1eb8f1b2 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -997,6 +997,9 @@(LabelAddress (dst WritableReg)(label MachLabel))+ ;; A pseudoinstruction that serves as a sequence point.+ (SequencePoint)+;; Emits an inline stack probe loop.;;;; Note that this is emitted post-regalloc so `start` and `end` can be@@ -5203,3 +5206,8 @@(let ((dst WritableReg (temp_writable_reg $I64))(_ Unit (emit (MInst.LabelAddress dst label))))dst))++;; Helper for creating a `SequencePoint` instruction.+(decl a64_sequence_point () SideEffectNoResult)+(rule (a64_sequence_point)+ (SideEffectNoResult.Inst (MInst.SequencePoint)))diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex db910a650a..91beb84efb 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -3547,6 +3547,10 @@ impl MachInstEmit for Inst {sink.use_label_at_offset(offset, label, LabelUse::Adr21);}+ &Inst::SequencePoint { .. } => {+ // Nothing.+ }+&Inst::StackProbeLoop { start, end, step } => {assert!(emit_info.0.enable_probestack());diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex dbff1df1b6..1ba8ec50ff 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -919,6 +919,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {Inst::LabelAddress { dst, .. } => {collector.reg_def(dst);}+ Inst::SequencePoint { .. } => {}Inst::StackProbeLoop { start, end, .. } => {collector.reg_early_def(start);collector.reg_use(end);@@ -2893,6 +2894,9 @@ impl Inst {let dst = pretty_print_reg(dst.to_reg());format!("label_address {dst}, {label:?}")}+ &Inst::SequencePoint {} => {+ format!("sequence_point")+ }&Inst::StackProbeLoop { start, end, step } => {let start = pretty_print_reg(start.to_reg());let end = pretty_print_reg(end);diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex f304e5c620..c32edb2b1f 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -3271,3 +3271,9 @@(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(a64_label_address succ_label)))++;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++(rule (lower (sequence_point))+ (side_effect+ (a64_sequence_point)))diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rsindex 98f4d5c7d6..892911edd8 100644--- a/cranelift/codegen/src/isa/aarch64/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/mod.rs@@ -74,11 +74,8 @@ impl TargetIsa for AArch64Backend {let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane);- let frame_size = emit_result.frame_size;let value_labels_ranges = emit_result.value_labels_ranges;let buffer = emit_result.buffer;- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;if let Some(disasm) = emit_result.disasm.as_ref() {log::debug!("disassembly:\n{disasm}");@@ -86,11 +83,8 @@ impl TargetIsa for AArch64Backend {Ok(CompiledCodeStencil {buffer,- frame_size,vcode: emit_result.disasm,value_labels_ranges,- sized_stackslot_offsets,- dynamic_stackslot_offsets,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,})diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isleindex 4b008a8867..901753841c 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle@@ -95,6 +95,9 @@;; A pseudoinstruction that loads the address of a label.(LabelAddress (dst WritableXReg)(label MachLabel))++ ;; A pseudoinstruction that serves as a sequence point.+ (SequencePoint)))@@ -792,3 +795,8 @@(let ((dst WritableReg (temp_writable_xreg))(_ Unit (emit (MInst.LabelAddress dst label))))dst))++;; Helper for creating a `SequencePoint` instruction.+(decl pulley_sequence_point () SideEffectNoResult)+(rule (pulley_sequence_point)+ (SideEffectNoResult.Inst (MInst.SequencePoint)))diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rsindex 08d8b5eaf7..c8de063259 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs+++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs@@ -596,6 +596,10 @@ fn pulley_emit<P>(let end = sink.cur_offset();sink.use_label_at_offset(end - 4, *label, LabelUse::PcRel);}++ Inst::SequencePoint { .. } => {+ // Nothing.+ }}}diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rsindex bc2490f7bc..606ec93ae3 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs+++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs@@ -335,6 +335,8 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {Inst::LabelAddress { dst, label: _ } => {collector.reg_def(dst);}++ Inst::SequencePoint { .. } => {}}}@@ -834,6 +836,10 @@ impl Inst {let dst = format_reg(dst.to_reg().to_reg());format!("label_address {dst}, {label:?}")}++ Inst::SequencePoint {} => {+ format!("sequence_point")+ }}}}diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isleindex 12770d844e..7cde5f2027 100644--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle@@ -1836,3 +1836,9 @@(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(pulley_label_address succ_label)))++;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++(rule (lower (sequence_point))+ (side_effect+ (pulley_sequence_point)))diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rsindex 0b2cf027f0..c3f16efa2b 100644--- a/cranelift/codegen/src/isa/pulley_shared/mod.rs+++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs@@ -180,11 +180,8 @@ wherelet want_disasm =want_disasm || (cfg!(feature = "trace-log") && log::log_enabled!(log::Level::Debug));let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane);- let frame_size = emit_result.frame_size;let value_labels_ranges = emit_result.value_labels_ranges;let buffer = emit_result.buffer;- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;if let Some(disasm) = emit_result.disasm.as_ref() {log::debug!("disassembly:\n{disasm}");@@ -192,11 +189,8 @@ whereOk(CompiledCodeStencil {buffer,- frame_size,vcode: emit_result.disasm,value_labels_ranges,- sized_stackslot_offsets,- dynamic_stackslot_offsets,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,})diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isleindex f118fcf710..62774b59fe 100644--- a/cranelift/codegen/src/isa/riscv64/inst.isle+++ b/cranelift/codegen/src/isa/riscv64/inst.isle@@ -360,6 +360,9 @@(EmitIsland;; The needed space before the next deadline.(needed_space u32))++ ;; A pseudoinstruction that serves as a sequence point.+ (SequencePoint)))(type AtomicOP (enum@@ -3262,3 +3265,8 @@(let ((dst WritableReg (temp_writable_reg $I64))(_ Unit (emit (MInst.LabelAddress dst label))))dst))++;; Helper for creating a `SequencePoint` instruction.+(decl rv64_sequence_point () SideEffectNoResult)+(rule (rv64_sequence_point)+ (SideEffectNoResult.Inst (MInst.SequencePoint)))diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rsindex e7a64b605f..77bd801c4f 100644--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs@@ -195,6 +195,7 @@ impl Inst {| Inst::Unwind { .. }| Inst::DummyUse { .. }| Inst::LabelAddress { .. }+ | Inst::SequencePoint { .. }| Inst::Popcnt { .. }| Inst::Cltz { .. }| Inst::Brev8 { .. }@@ -2769,6 +2770,10 @@ impl Inst {sink.bind_label(jump_around_label, &mut state.ctrl_plane);}}++ Inst::SequencePoint { .. } => {+ // Nothing.+ }}}}diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rsindex 5f4c037b29..4a3890b093 100644--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs@@ -699,6 +699,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {Inst::LabelAddress { dst, .. } => {collector.reg_def(dst);}+ Inst::SequencePoint { .. } => {}}}@@ -1680,6 +1681,10 @@ impl Inst {let dst = format_reg(dst.to_reg());format!("label_address {dst}, {label:?}")}++ Inst::SequencePoint {} => {+ format!("sequence_point")+ }}}}diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isleindex d2329133b7..920e5c2119 100644--- a/cranelift/codegen/src/isa/riscv64/lower.isle+++ b/cranelift/codegen/src/isa/riscv64/lower.isle@@ -3138,3 +3138,9 @@(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(rv64_label_address succ_label)))++;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++(rule (lower (sequence_point))+ (side_effect+ (rv64_sequence_point)))diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rsindex 3ae2b6dfb3..873b698510 100644--- a/cranelift/codegen/src/isa/riscv64/mod.rs+++ b/cranelift/codegen/src/isa/riscv64/mod.rs@@ -75,11 +75,8 @@ impl TargetIsa for Riscv64Backend {let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug);let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane);- let frame_size = emit_result.frame_size;let value_labels_ranges = emit_result.value_labels_ranges;let buffer = emit_result.buffer;- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;if let Some(disasm) = emit_result.disasm.as_ref() {log::debug!("disassembly:\n{disasm}");@@ -87,11 +84,8 @@ impl TargetIsa for Riscv64Backend {Ok(CompiledCodeStencil {buffer,- frame_size,vcode: emit_result.disasm,value_labels_ranges,- sized_stackslot_offsets,- dynamic_stackslot_offsets,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,})diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isleindex 91d0227b3c..46576a42d4 100644--- a/cranelift/codegen/src/isa/s390x/inst.isle+++ b/cranelift/codegen/src/isa/s390x/inst.isle@@ -1019,6 +1019,9 @@;; A pseudoinstruction that loads the address of a label.(LabelAddress (dst WritableReg)(label MachLabel))++ ;; A pseudoinstruction that serves as a sequence point.+ (SequencePoint)));; Primitive types used in instruction formats.@@ -4941,6 +4944,11 @@(_ Unit (emit (MInst.LabelAddress dst label))))dst))+;; Helper for creating a `SequencePoint` instruction.+(decl s390x_sequence_point () SideEffectNoResult)+(rule (s390x_sequence_point)+ (SideEffectNoResult.Inst (MInst.SequencePoint)))+;; Implicit conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(convert WritableRegPair RegPair writable_regpair_to_regpair)diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rsindex 98718d1ee1..d67bd5f56c 100644--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs@@ -3544,6 +3544,10 @@ impl Inst {};inst.emit(sink, emit_info, state);}++ &Inst::SequencePoint { .. } => {+ // Nothing.+ }}state.clear_post_insn();diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rsindex 256de41504..8c16117ec6 100644--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs@@ -284,6 +284,8 @@ impl Inst {Inst::DummyUse { .. } => InstructionSet::Base,Inst::LabelAddress { .. } => InstructionSet::Base,++ Inst::SequencePoint { .. } => InstructionSet::Base,}}@@ -1011,6 +1013,7 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl OpeInst::LabelAddress { dst, .. } => {collector.reg_def(dst);}+ Inst::SequencePoint { .. } => {}}}@@ -3414,6 +3417,9 @@ impl Inst {let dst = pretty_print_reg(dst.to_reg());format!("label_address {dst}, {label:?}")}+ &Inst::SequencePoint {} => {+ format!("sequence_point")+ }}}}diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isleindex b86358749c..41d8507f6d 100644--- a/cranelift/codegen/src/isa/s390x/lower.isle+++ b/cranelift/codegen/src/isa/s390x/lower.isle@@ -4073,3 +4073,9 @@(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(s390x_label_address succ_label)))++;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++(rule (lower (sequence_point))+ (side_effect+ (s390x_sequence_point)))diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rsindex 0434c4fcba..8768b4eebf 100644--- a/cranelift/codegen/src/isa/s390x/mod.rs+++ b/cranelift/codegen/src/isa/s390x/mod.rs@@ -74,11 +74,8 @@ impl TargetIsa for S390xBackend {let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;let emit_result = vcode.emit(®alloc_result, want_disasm, flags, ctrl_plane);- let frame_size = emit_result.frame_size;let value_labels_ranges = emit_result.value_labels_ranges;let buffer = emit_result.buffer;- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;if let Some(disasm) = emit_result.disasm.as_ref() {log::debug!("disassembly:\n{disasm}");@@ -86,11 +83,8 @@ impl TargetIsa for S390xBackend {Ok(CompiledCodeStencil {buffer,- frame_size,vcode: emit_result.disasm,value_labels_ranges,- sized_stackslot_offsets,- dynamic_stackslot_offsets,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,})diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex c8ea499e6e..c5c1c7c891 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -320,6 +320,9 @@(LabelAddress (dst WritableGpr)(label MachLabel))+ ;; A pseudoinstruction that serves as a sequence point.+ (SequencePoint)+;; An instruction assembled outside of cranelift-codegen.(External (inst AssemblerInst))))@@ -4097,3 +4100,8 @@(decl libcall_3 (LibCall Reg Reg Reg) Reg)(extern constructor libcall_3 libcall_3)++;; Helper for creating a `SequencePoint` instruction.+(decl x64_sequence_point () SideEffectNoResult)+(rule (x64_sequence_point)+ (SideEffectNoResult.Inst (MInst.SequencePoint)))diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rsindex 3f1582ce8d..a922cf7377 100644--- a/cranelift/codegen/src/isa/x64/inst/emit.rs+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs@@ -1815,6 +1815,10 @@ pub(crate) fn emit(asm::inst::leaq_rm::new(*dst, Amode::rip_relative(*label)).emit(sink, info, state);}+ Inst::SequencePoint { .. } => {+ // Nothing.+ }+Inst::External { inst } => {let frame = state.frame_layout();emit_maybe_shrink(diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex f01dd8781b..c3ee37a958 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -106,7 +106,8 @@ impl Inst {| Inst::CoffTlsGetAddr { .. }| Inst::Unwind { .. }| Inst::DummyUse { .. }- | Inst::LabelAddress { .. } => true,+ | Inst::LabelAddress { .. }+ | Inst::SequencePoint => true,Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => emit_info.cmpxchg16b(),@@ -828,6 +829,10 @@ impl PrettyPrint for Inst {format!("label_address {dst}, {label:?}")}+ Inst::SequencePoint {} => {+ format!("sequence_point")+ }+Inst::External { inst } => {format!("{inst}")}@@ -1193,6 +1198,8 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_def(dst);}+ Inst::SequencePoint { .. } => {}+Inst::External { inst } => {inst.visit(&mut external::RegallocVisitor { collector });}diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 1a051f41fd..ebfd43bbf1 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -5059,3 +5059,9 @@(rule (lower (nop))(invalid_reg))++;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++(rule (lower (sequence_point))+ (side_effect+ (x64_sequence_point)))diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rsindex f40fef58c0..0edaf5ccb7 100644--- a/cranelift/codegen/src/isa/x64/mod.rs+++ b/cranelift/codegen/src/isa/x64/mod.rs@@ -83,11 +83,8 @@ impl TargetIsa for X64Backend {let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane);- let frame_size = emit_result.frame_size;let value_labels_ranges = emit_result.value_labels_ranges;let buffer = emit_result.buffer;- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;if let Some(disasm) = emit_result.disasm.as_ref() {crate::trace!("disassembly:\n{}", disasm);@@ -95,11 +92,8 @@ impl TargetIsa for X64Backend {Ok(CompiledCodeStencil {buffer,- frame_size,vcode: emit_result.disasm,value_labels_ranges,- sized_stackslot_offsets,- dynamic_stackslot_offsets,bb_starts: emit_result.bb_offsets,bb_edges: emit_result.bb_edges,})diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rsindex 75a0c8e3d2..09b356d699 100644--- a/cranelift/codegen/src/isa/x64/pcc.rs+++ b/cranelift/codegen/src/isa/x64/pcc.rs@@ -211,6 +211,8 @@ pub(crate) fn check(Inst::LabelAddress { .. } => Err(PccError::UnimplementedInst),+ Inst::SequencePoint { .. } => Ok(()),+Inst::External { .. } => Ok(()), // TODO: unsure what to do about this!}}diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rsindex 64d37df76a..425edb25e1 100644--- a/cranelift/codegen/src/lib.rs+++ b/cranelift/codegen/src/lib.rs@@ -68,9 +68,10 @@ pub use crate::machinst::buffer::{OpenPatchRegion, PatchRegion,};pub use crate::machinst::{- CallInfo, CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit,- MachInstEmitState, MachLabel, RealReg, Reg, RelocDistance, TextSectionBuilder, VCodeConstant,- VCodeConstantData, VCodeConstants, VCodeInst, Writable,+ CallInfo, CompiledCode, Final, MachBuffer, MachBufferDebugTagList, MachBufferFinalized,+ MachBufferFrameLayout, MachDebugTagPos, MachInst, MachInstEmit, MachInstEmitState, MachLabel,+ RealReg, Reg, RelocDistance, TextSectionBuilder, VCodeConstant, VCodeConstantData,+ VCodeConstants, VCodeInst, Writable,};mod alias_analysis;diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rsindex cc13a6d49d..91510e5a46 100644--- a/cranelift/codegen/src/machinst/abi.rs+++ b/cranelift/codegen/src/machinst/abi.rs@@ -100,8 +100,8 @@use crate::CodegenError;use crate::entity::SecondaryMap;-use crate::ir::types::*;use crate::ir::{ArgumentExtension, ArgumentPurpose, ExceptionTag, Signature};+use crate::ir::{StackSlotKey, types::*};use crate::isa::TargetIsa;use crate::settings::ProbestackStrategy;use crate::{ir, isa};@@ -1142,6 +1142,8 @@ pub struct Callee<M: ABIMachineSpec> {dynamic_stackslots: PrimaryMap<DynamicStackSlot, u32>,/// Offsets to each sized stackslot.sized_stackslots: PrimaryMap<StackSlot, u32>,+ /// Descriptors for sized stackslots.+ sized_stackslot_keys: SecondaryMap<StackSlot, Option<StackSlotKey>>,/// Total stack size of all stackslotsstackslots_size: u32,/// Stack size to be reserved for outgoing arguments.@@ -1227,6 +1229,7 @@ impl<M: ABIMachineSpec> Callee<M> {// Compute sized stackslot locations and total stackslot size.let mut end_offset: u32 = 0;let mut sized_stackslots = PrimaryMap::new();+ let mut sized_stackslot_keys = SecondaryMap::new();for (stackslot, data) in f.sized_stack_slots.iter() {// We start our computation possibly unaligned where the previous@@ -1250,6 +1253,7 @@ impl<M: ABIMachineSpec> Callee<M> {debug_assert_eq!(stackslot.as_u32() as usize, sized_stackslots.len());sized_stackslots.push(start_offset);+ sized_stackslot_keys[stackslot] = data.key;}// Compute dynamic stackslot locations and total stackslot size.@@ -1304,6 +1308,7 @@ impl<M: ABIMachineSpec> Callee<M> {dynamic_stackslots,dynamic_type_sizes,sized_stackslots,+ sized_stackslot_keys,stackslots_size,outgoing_args_size: 0,tail_args_size,@@ -2324,22 +2329,29 @@ impl<M: ABIMachineSpec> Callee<M> {.expect("frame layout not computed before prologue generation")}- /// Returns the full frame size for the given function, after prologue- /// emission has run. This comprises the spill slots and stack-storage- /// slots as well as storage for clobbered callee-save registers, but- /// not arguments arguments pushed at callsites within this function,- /// or other ephemeral pushes.- pub fn frame_size(&self) -> u32 {+ /// Returns the offset from SP to FP for the given function, after+ /// the prologue has set up the frame. This comprises the spill+ /// slots and stack-storage slots as well as storage for clobbered+ /// callee-save registers and outgoing arguments at callsites+ /// (space for which is reserved during frame setup).+ pub fn sp_to_fp_offset(&self) -> u32 {let frame_layout = self.frame_layout();- frame_layout.clobber_size + frame_layout.fixed_frame_storage_size+ frame_layout.clobber_size+ + frame_layout.fixed_frame_storage_size+ + frame_layout.outgoing_args_size}/// Returns offset from the slot base in the current frame to the caller's SP.pub fn slot_base_to_caller_sp_offset(&self) -> u32 {+ // Note: this looks very similar to `frame_size()` above, but+ // it differs in both endpoints: it measures from the bottom+ // of stackslots, excluding outgoing args; and it includes the+ // setup area (FP/LR) size and any extra tail-args space.let frame_layout = self.frame_layout();frame_layout.clobber_size+ frame_layout.fixed_frame_storage_size+ frame_layout.setup_area_size+ + (frame_layout.tail_args_size - frame_layout.incoming_args_size)}/// Returns the size of arguments expected on the stack.@@ -2390,6 +2402,28 @@ impl<M: ABIMachineSpec> Callee<M> {let from = StackAMode::Slot(sp_off);<M>::gen_load_stack(from, to_reg.map(Reg::from), ty)}++ /// Provide metadata to be emitted alongside machine code.+ ///+ /// This metadata describes the frame layout sufficiently to find+ /// stack slots, so that runtimes and unwinders can observe state+ /// set up by compiled code in stackslots allocated for that+ /// purpose.+ pub fn frame_slot_metadata(&self) -> MachBufferFrameLayout {+ let frame_to_fp_offset = self.sp_to_fp_offset();+ let mut stackslots = SecondaryMap::with_capacity(self.sized_stackslots.len());+ let storage_area_base = self.frame_layout().outgoing_args_size;+ for (slot, storage_area_offset) in &self.sized_stackslots {+ stackslots[slot] = MachBufferStackSlot {+ offset: storage_area_base.checked_add(*storage_area_offset).unwrap(),+ key: self.sized_stackslot_keys[slot],+ };+ }+ MachBufferFrameLayout {+ frame_to_fp_offset,+ stackslots,+ }+ }}/// An input argument to a call instruction: the vreg that is used,diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rsindex b489b7c079..444ae2a25b 100644--- a/cranelift/codegen/src/machinst/buffer.rs+++ b/cranelift/codegen/src/machinst/buffer.rs@@ -172,7 +172,7 @@use crate::binemit::{Addend, CodeOffset, Reloc};use crate::ir::function::FunctionParameters;-use crate::ir::{ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode};+use crate::ir::{DebugTag, ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode};use crate::isa::unwind::UnwindInst;use crate::machinst::{BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst,@@ -182,7 +182,7 @@ use crate::{MachInstEmitState, ir};use crate::{VCodeConstantData, timing};use core::ops::Range;use cranelift_control::ControlPlane;-use cranelift_entity::{PrimaryMap, entity_impl};+use cranelift_entity::{PrimaryMap, SecondaryMap, entity_impl};use smallvec::SmallVec;use std::cmp::Ordering;use std::collections::BinaryHeap;@@ -254,6 +254,10 @@ pub struct MachBuffer<I: VCodeInst> {exception_handlers: SmallVec<[MachExceptionHandler; 16]>,/// Any source location mappings referring to this code.srclocs: SmallVec<[MachSrcLoc<Stencil>; 64]>,+ /// Any debug tags referring to this code.+ debug_tags: Vec<MachDebugTags>,+ /// Pool of debug tags referenced by `MachDebugTags` entries.+ debug_tag_pool: Vec<DebugTag>,/// Any user stack maps for this code.////// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted@@ -322,6 +326,10 @@ pub struct MachBuffer<I: VCodeInst> {/// Indicates when a patchable region is currently open, to guard that it's/// not possible to nest patchable regions.open_patchable: bool,+ /// Stack frame layout metadata. If provided for a MachBuffer+ /// containing a function body, this allows interpretation of+ /// runtime state given a view of an active stack frame.+ frame_layout: Option<MachBufferFrameLayout>,}impl MachBufferFinalized<Stencil> {@@ -338,9 +346,12 @@ impl MachBufferFinalized<Stencil> {.into_iter().map(|srcloc| srcloc.apply_base_srcloc(base_srcloc)).collect(),+ debug_tags: self.debug_tags,+ debug_tag_pool: self.debug_tag_pool,user_stack_maps: self.user_stack_maps,unwind_info: self.unwind_info,alignment: self.alignment,+ frame_layout: self.frame_layout,}}}@@ -367,11 +378,19 @@ pub struct MachBufferFinalized<T: CompilePhase> {pub(crate) exception_handlers: SmallVec<[FinalizedMachExceptionHandler; 16]>,/// Any source location mappings referring to this code.pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>,+ /// Any debug tags referring to this code.+ pub(crate) debug_tags: Vec<MachDebugTags>,+ /// Pool of debug tags referenced by `MachDebugTags` entries.+ pub(crate) debug_tag_pool: Vec<DebugTag>,/// Any user stack maps for this code.////// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted/// by code offset, and each stack map covers `span` bytes on the stack.pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,+ /// Stack frame layout metadata. If provided for a MachBuffer+ /// containing a function body, this allows interpretation of+ /// runtime state given a view of an active stack frame.+ pub(crate) frame_layout: Option<MachBufferFrameLayout>,/// Any unwind info at a given location.pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,/// The required alignment of this buffer.@@ -447,6 +466,8 @@ impl<I: VCodeInst> MachBuffer<I> {call_sites: SmallVec::new(),exception_handlers: SmallVec::new(),srclocs: SmallVec::new(),+ debug_tags: vec![],+ debug_tag_pool: vec![],user_stack_maps: SmallVec::new(),unwind_info: SmallVec::new(),cur_srcloc: None,@@ -464,6 +485,7 @@ impl<I: VCodeInst> MachBuffer<I> {constants: Default::default(),used_constants: Default::default(),open_patchable: false,+ frame_layout: None,}}@@ -838,6 +860,8 @@ impl<I: VCodeInst> MachBuffer<I> {// (end of buffer)self.data.truncate(b.start as usize);self.pending_fixup_records.truncate(b.fixup);++ // Trim srclocs and debug tags now past the end of the buffer.while let Some(last_srcloc) = self.srclocs.last_mut() {if last_srcloc.end <= b.start {break;@@ -848,6 +872,13 @@ impl<I: VCodeInst> MachBuffer<I> {}self.srclocs.pop();}+ while let Some(last_debug_tag) = self.debug_tags.last() {+ if last_debug_tag.offset <= b.start {+ break;+ }+ self.debug_tags.pop();+ }+// State:// [PRE CODE]// cur_off, Offset b.start, b.labels_at_this_branch:@@ -1535,9 +1566,12 @@ impl<I: VCodeInst> MachBuffer<I> {call_sites: self.call_sites,exception_handlers: finalized_exception_handlers,srclocs,+ debug_tags: self.debug_tags,+ debug_tag_pool: self.debug_tag_pool,user_stack_maps: self.user_stack_maps,unwind_info: self.unwind_info,alignment,+ frame_layout: self.frame_layout,}}@@ -1693,6 +1727,19 @@ impl<I: VCodeInst> MachBuffer<I> {self.user_stack_maps.push((return_addr, span, stack_map));}+ /// Push a debug tag associated with the current buffer offset.+ pub fn push_debug_tags(&mut self, pos: MachDebugTagPos, tags: &[DebugTag]) {+ trace!("debug tags at offset {}: {tags:?}", self.cur_offset());+ let start = u32::try_from(self.debug_tag_pool.len()).unwrap();+ self.debug_tag_pool.extend(tags.iter().cloned());+ let end = u32::try_from(self.debug_tag_pool.len()).unwrap();+ self.debug_tags.push(MachDebugTags {+ offset: self.cur_offset(),+ pos,+ range: start..end,+ });+ }+/// Increase the alignment of the buffer to the given alignment if bigger/// than the current alignment.pub fn set_log2_min_function_alignment(&mut self, align_to: u8) {@@ -1701,6 +1748,12 @@ impl<I: VCodeInst> MachBuffer<I> {.expect("log2_min_function_alignment too large"),);}++ /// Set the frame layout metadata.+ pub fn set_frame_layout(&mut self, frame_layout: MachBufferFrameLayout) {+ debug_assert!(self.frame_layout.is_none());+ self.frame_layout = Some(frame_layout);+ }}impl<I: VCodeInst> Extend<u8> for MachBuffer<I> {@@ -1717,6 +1770,19 @@ impl<T: CompilePhase> MachBufferFinalized<T> {&self.srclocs[..]}+ /// Get all debug tags, sorted by associated offset.+ pub fn debug_tags(&self) -> impl Iterator<Item = MachBufferDebugTagList<'_>> {+ self.debug_tags.iter().map(|tags| {+ let start = usize::try_from(tags.range.start).unwrap();+ let end = usize::try_from(tags.range.end).unwrap();+ MachBufferDebugTagList {+ offset: tags.offset,+ pos: tags.pos,+ tags: &self.debug_tag_pool[start..end],+ }+ })+ }+/// Get the total required size for the code.pub fn total_size(&self) -> CodeOffset {self.data.len() as CodeOffset@@ -1792,6 +1858,11 @@ impl<T: CompilePhase> MachBufferFinalized<T> {}})}++ /// Get the frame layout, if known.+ pub fn frame_layout(&self) -> Option<&MachBufferFrameLayout> {+ self.frame_layout.as_ref()+ }}/// An item in the exception-handler list for a callsite, with label@@ -2117,6 +2188,118 @@ impl MachBranch {}}+/// Stack-frame layout information carried through to machine+/// code. This provides sufficient information to interpret an active+/// stack frame from a running function, if provided.+#[derive(Clone, Debug, PartialEq)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub struct MachBufferFrameLayout {+ /// Offset from bottom of frame to FP (near top of frame). This+ /// allows reading the frame given only FP.+ pub frame_to_fp_offset: u32,+ /// Offset from bottom of frame for each StackSlot,+ pub stackslots: SecondaryMap<ir::StackSlot, MachBufferStackSlot>,+}++/// Descriptor for a single stack slot in the compiled function.+#[derive(Clone, Debug, PartialEq, Default)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub struct MachBufferStackSlot {+ /// Offset from the bottom of the stack frame.+ pub offset: u32,++ /// User-provided key to describe this stack slot.+ pub key: Option<ir::StackSlotKey>,+}++/// Debug tags: a sequence of references to a stack slot, or a+/// user-defined value, at a particular PC.+#[derive(Clone, Debug, PartialEq)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub(crate) struct MachDebugTags {+ /// Offset at which this tag applies.+ pub offset: CodeOffset,++ /// Position on the attached instruction. This indicates whether+ /// the tags attach to the prior instruction (i.e., as a return+ /// point from a call) or the current instruction (i.e., as a PC+ /// seen during a trap).+ pub pos: MachDebugTagPos,++ /// The range in the tag pool.+ pub range: Range<u32>,+}++/// Debug tag position on an instruction.+///+/// We need to distinguish position on an instruction, and not just+/// use offsets, because of the following case:+///+/// ```plain+/// <tag1, tag2> call ...+/// <tag3, tag4> trapping_store ...+/// ```+///+/// If the stack is walked and interpreted with debug tags while+/// within the call, the PC seen will be the return point, i.e. the+/// address after the call. If the stack is walked and interpreted+/// with debug tags upon a trap of the following instruction, it will+/// be the PC of that instruction -- which is the same PC! Thus to+/// disambiguate which tags we want, we attach a "pre/post" flag to+/// every group of tags at an offset; and when we look up tags, we+/// look them up for an offset and "position" at that offset.+///+/// Thus there are logically two positions at every offset -- so the+/// above will be emitted as+///+/// ```plain+/// 0: call ...+/// 4, post: <tag1, tag2>+/// 4, pre: <tag3, tag4>+/// 4: trapping_store ...+/// ```+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub enum MachDebugTagPos {+ /// Tags attached after the instruction that ends at this offset.+ ///+ /// This is used to attach tags to a call, because the PC we see+ /// when walking the stack is the *return point*.+ Post,+ /// Tags attached before the instruction that starts at this offset.+ ///+ /// This is used to attach tags to every other kind of+ /// instruction, because the PC we see when processing a trap of+ /// that instruction is the PC of that instruction, not the+ /// following one.+ Pre,+}++/// Iterator item for visiting debug tags.+pub struct MachBufferDebugTagList<'a> {+ /// Offset at which this tag applies.+ pub offset: CodeOffset,++ /// Position at this offset ("post", attaching to prior+ /// instruction, or "pre", attaching to next instruction).+ pub pos: MachDebugTagPos,++ /// The underlying tags.+ pub tags: &'a [DebugTag],+}+/// Implementation of the `TextSectionBuilder` trait backed by `MachBuffer`.////// Note that `MachBuffer` was primarily written for intra-function referencesdiff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rsindex 90007f25f9..47eec0767e 100644--- a/cranelift/codegen/src/machinst/lower.rs+++ b/cranelift/codegen/src/machinst/lower.rs@@ -873,6 +873,21 @@ impl<'func, I: VCodeInst> Lower<'func, I> {}}+ // If the CLIF instruction had debug tags, copy them to+ // the VCode. Place on all VCode instructions lowered from+ // this CLIF instruction.+ let debug_tags = self.f.debug_tags.get(inst);+ if !debug_tags.is_empty() && self.vcode.vcode.num_insts() > 0 {+ let end = self.vcode.vcode.num_insts();+ for i in start..end {+ let backwards_index = BackwardsInsnIndex::new(i);+ log::trace!(+ "debug tags on {inst}; associating {debug_tags:?} with {backwards_index:?}"+ );+ self.vcode.add_debug_tags(backwards_index, debug_tags);+ }+ }+// maybe insert random instructionif ctrl_plane.get_decision() {if ctrl_plane.get_decision() {diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rsindex 38355b8cbd..77033e1c15 100644--- a/cranelift/codegen/src/machinst/mod.rs+++ b/cranelift/codegen/src/machinst/mod.rs@@ -389,16 +389,10 @@ pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {pub struct CompiledCodeBase<T: CompilePhase> {/// Machine code.pub buffer: MachBufferFinalized<T>,- /// Size of stack frame, in bytes.- pub frame_size: u32,/// Disassembly, if requested.pub vcode: Option<String>,/// Debug info: value labels to registers/stackslots at code offsets.pub value_labels_ranges: ValueLabelsRanges,- /// Debug info: stackslots to stack pointer offsets.- pub sized_stackslot_offsets: PrimaryMap<StackSlot, u32>,- /// Debug info: stackslots to stack pointer offsets.- pub dynamic_stackslot_offsets: PrimaryMap<DynamicStackSlot, u32>,/// Basic-block layout info: block start offsets.////// This info is generated only if the `machine_code_cfg_info`@@ -418,11 +412,8 @@ impl CompiledCodeStencil {pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode {CompiledCode {buffer: self.buffer.apply_base_srcloc(params.base_srcloc()),- frame_size: self.frame_size,vcode: self.vcode,value_labels_ranges: self.value_labels_ranges,- sized_stackslot_offsets: self.sized_stackslot_offsets,- dynamic_stackslot_offsets: self.dynamic_stackslot_offsets,bb_starts: self.bb_starts,bb_edges: self.bb_edges,}diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rsindex cd6b8294ba..9bb44e43d5 100644--- a/cranelift/codegen/src/machinst/vcode.rs+++ b/cranelift/codegen/src/machinst/vcode.rs@@ -34,6 +34,7 @@ use rustc_hash::FxHashMap;use core::cmp::Ordering;use core::fmt::{self, Write};use core::mem::take;+use core::ops::Range;use cranelift_entity::{Keys, entity_impl};use std::collections::HashMap;use std::collections::hash_map::Entry;@@ -103,6 +104,15 @@ pub struct VCode<I: VCodeInst> {/// user stack map.user_stack_maps: FxHashMap<BackwardsInsnIndex, ir::UserStackMap>,+ /// A map from backwards instruction index to the debug tags for+ /// that instruction. Each entry indexes a range in the+ /// `debug_tag_pool`.+ debug_tags: FxHashMap<BackwardsInsnIndex, Range<u32>>,++ /// Pooled storage for sequences of debug tags; indexed by entries+ /// in `debug_tags`.+ debug_tag_pool: Vec<ir::DebugTag>,+/// Operands: pre-regalloc references to virtual registers with/// constraints, in one flattened array. This allows the regalloc/// to efficiently access all operands without requiring expensive@@ -221,17 +231,8 @@ pub struct EmitResult {/// epilogue(s), and makes use of the regalloc results.pub disasm: Option<String>,- /// Offsets of sized stackslots.- pub sized_stackslot_offsets: PrimaryMap<StackSlot, u32>,-- /// Offsets of dynamic stackslots.- pub dynamic_stackslot_offsets: PrimaryMap<DynamicStackSlot, u32>,-/// Value-labels information (debug metadata).pub value_labels_ranges: ValueLabelsRanges,-- /// Stack frame size.- pub frame_size: u32,}/// A builder for a VCode function body.@@ -617,6 +618,14 @@ impl<I: VCodeInst> VCodeBuilder<I> {let old_entry = self.vcode.user_stack_maps.insert(inst, stack_map);debug_assert!(old_entry.is_none());}++ /// Add debug tags for the associated instruction.+ pub fn add_debug_tags(&mut self, inst: BackwardsInsnIndex, entries: &[ir::DebugTag]) {+ let start = u32::try_from(self.vcode.debug_tag_pool.len()).unwrap();+ self.vcode.debug_tag_pool.extend(entries.iter().cloned());+ let end = u32::try_from(self.vcode.debug_tag_pool.len()).unwrap();+ self.vcode.debug_tags.insert(inst, start..end);+ }}const NO_INST_OFFSET: CodeOffset = u32::MAX;@@ -637,6 +646,8 @@ impl<I: VCodeInst> VCode<I> {vreg_types: vec![],insts: Vec::with_capacity(10 * n_blocks),user_stack_maps: FxHashMap::default(),+ debug_tags: FxHashMap::default(),+ debug_tag_pool: vec![],operands: Vec::with_capacity(30 * n_blocks),operand_ranges: Ranges::with_capacity(10 * n_blocks),clobbers: FxHashMap::default(),@@ -938,8 +949,11 @@ impl<I: VCodeInst> VCode<I> {// function.let index = iix.to_backwards_insn_index(self.num_insts());let user_stack_map = self.user_stack_maps.remove(&index);- let user_stack_map_disasm =- user_stack_map.as_ref().map(|m| format!(" ; {m:?}"));+ let user_stack_map_disasm = if want_disasm {+ user_stack_map.as_ref().map(|m| format!(" ; {m:?}"))+ } else {+ None+ };(user_stack_map, user_stack_map_disasm)};@@ -950,11 +964,50 @@ impl<I: VCodeInst> VCode<I> {None};+ // Place debug tags in the emission buffer+ // either at the offset prior to the+ // instruction or after the instruction,+ // depending on whether this is a call. See+ // the documentation on [`MachDebugTagPos`]+ // for details on why.+ let mut debug_tag_disasm = None;+ let mut place_debug_tags =+ |this: &VCode<I>, pos: MachDebugTagPos, buffer: &mut MachBuffer<I>| {+ // As above, translate the forward instruction+ // index to a backward index for the lookup.+ let debug_tag_range = {+ let index = iix.to_backwards_insn_index(this.num_insts());+ this.debug_tags.get(&index)+ };+ if let Some(range) = debug_tag_range {+ let start = usize::try_from(range.start).unwrap();+ let end = usize::try_from(range.end).unwrap();+ let tags = &this.debug_tag_pool[start..end];++ if want_disasm {+ debug_tag_disasm =+ Some(format!(" ; ^-- debug @ {pos:?}: {tags:?}"));+ }+ buffer.push_debug_tags(pos, tags);+ }+ };+ let debug_tag_pos =+ if self.insts[iix.index()].call_type() == CallType::Regular {+ MachDebugTagPos::Post+ } else {+ MachDebugTagPos::Pre+ };++ if debug_tag_pos == MachDebugTagPos::Pre {+ place_debug_tags(&self, debug_tag_pos, &mut buffer);+ }+// If the instruction we are about to emit is// a return, place an epilogue at this point// (and don't emit the return; the actual// epilogue will contain it).if self.insts[iix.index()].is_term() == MachTerminator::Ret {+ log::trace!("emitting epilogue");for inst in self.abi.gen_epilogue() {do_emit(&inst, &mut disasm, &mut buffer, &mut state);}@@ -981,6 +1034,8 @@ impl<I: VCodeInst> VCode<I> {);debug_assert!(allocs.next().is_none());+ log::trace!("emitting: {:?}", self.insts[iix.index()]);+// Emit the instruction!do_emit(&self.insts[iix.index()],@@ -988,10 +1043,19 @@ impl<I: VCodeInst> VCode<I> {&mut buffer,&mut state,);++ if debug_tag_pos == MachDebugTagPos::Post {+ place_debug_tags(&self, debug_tag_pos, &mut buffer);+ }+if let Some(stack_map_disasm) = stack_map_disasm {disasm.push_str(&stack_map_disasm);disasm.push('\n');}+ if let Some(debug_tag_disasm) = debug_tag_disasm {+ disasm.push_str(&debug_tag_disasm);+ disasm.push('\n');+ }}}@@ -1113,17 +1177,16 @@ impl<I: VCodeInst> VCode<I> {self.monotonize_inst_offsets(&mut inst_offsets[..], func_body_len);let value_labels_ranges =self.compute_value_labels_ranges(regalloc, &inst_offsets[..], func_body_len);- let frame_size = self.abi.frame_size();++ // Store metadata about frame layout in the MachBuffer.+ buffer.set_frame_layout(self.abi.frame_slot_metadata());EmitResult {buffer: buffer.finish(&self.constants, ctrl_plane),bb_offsets,bb_edges,disasm: if want_disasm { Some(disasm) } else { None },- sized_stackslot_offsets: self.abi.sized_stackslot_offsets().clone(),- dynamic_stackslot_offsets: self.abi.dynamic_stackslot_offsets().clone(),value_labels_ranges,- frame_size,}}diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rsindex b91f1bcc04..fbb04cd771 100644--- a/cranelift/codegen/src/verifier/mod.rs+++ b/cranelift/codegen/src/verifier/mod.rs@@ -2025,6 +2025,21 @@ impl<'a> Verifier<'a> {Ok(())}+ pub fn debug_tags(&self, inst: Inst, errors: &mut VerifierErrors) -> VerifierStepResult {+ // Tags can only be present on calls and sequence points.+ let op = self.func.dfg.insts[inst].opcode();+ let tags_allowed = op.is_call() || op == Opcode::SequencePoint;+ let has_tags = self.func.debug_tags.has(inst);+ if has_tags && !tags_allowed {+ return errors.fatal((+ inst,+ "debug tags present on non-call, non-sequence point instruction".to_string(),+ ));+ }++ Ok(())+ }+pub fn run(&self, errors: &mut VerifierErrors) -> VerifierStepResult {self.verify_global_values(errors)?;self.verify_memory_types(errors)?;@@ -2043,6 +2058,7 @@ impl<'a> Verifier<'a> {self.typecheck(inst, errors)?;self.immediate_constraints(inst, errors)?;self.iconst_bounds(inst, errors)?;+ self.debug_tags(inst, errors)?;}self.encodable_as_bb(block, errors)?;diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rsindex d2fbce11bb..c0025aa6fc 100644--- a/cranelift/codegen/src/write.rs+++ b/cranelift/codegen/src/write.rs@@ -259,8 +259,12 @@ fn decorate_block<FW: FuncWriter>(aliases: &SecondaryMap<Value, Vec<Value>>,block: Block,) -> fmt::Result {- // Indent all instructions if any srclocs are present.- let indent = if func.rel_srclocs().is_empty() { 4 } else { 36 };+ // Indent all instructions if any srclocs or debug tags are present.+ let indent = if func.rel_srclocs().is_empty() && func.debug_tags.is_empty() {+ 4+ } else {+ 36+ };func_w.write_block_header(w, func, block, indent)?;for a in func.dfg.block_params(block).iter().cloned() {@@ -336,7 +340,7 @@ fn write_instruction(func: &Function,aliases: &SecondaryMap<Value, Vec<Value>>,inst: Inst,- indent: usize,+ mut indent: usize,) -> fmt::Result {// Prefix containing source location, encoding, and value locations.let mut s = String::with_capacity(16);@@ -347,6 +351,9 @@ fn write_instruction(write!(s, "{srcloc} ")?;}+ // Write out any debug tags.+ write_debug_tags(w, &func, inst, &mut indent)?;+// Write out prefix and indent the instruction.write!(w, "{s:indent$}")?;@@ -572,6 +579,26 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmtOk(())}+fn write_debug_tags(+ w: &mut dyn Write,+ func: &Function,+ inst: Inst,+ indent: &mut usize,+) -> fmt::Result {+ let tags = func.debug_tags.get(inst);+ if !tags.is_empty() {+ let tags = tags+ .iter()+ .map(|tag| format!("{tag}"))+ .collect::<Vec<_>>()+ .join(", ");+ let s = format!("<{tags}> ");+ write!(w, "{s}")?;+ *indent = indent.saturating_sub(s.len());+ }+ Ok(())+}+fn write_user_stack_map_entries(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt::Result {let entries = match dfg.user_stack_map_entries(inst) {None => return Ok(()),diff --git a/cranelift/filetests/filetests/inline/debug.clif b/cranelift/filetests/filetests/inline/debug.clifnew file mode 100644index 0000000000..a6820e3a33--- /dev/null+++ b/cranelift/filetests/filetests/inline/debug.clif@@ -0,0 +1,42 @@+test inline precise-output+target x86_64++function %f0(i32, i32) -> i32 {+ ss0 = explicit_slot 64, key = 0xfedc_ba98_7654_3210+block0(v0: i32, v1: i32):+<ss0, 1, 0> sequence_point+ v2 = iadd v0, v1+ return v2+}++; (no functions inlined into %f0)++function %f1() -> i32 {+ fn0 = %f0(i32, i32) -> i32+ ss0 = explicit_slot 64, key = 0x1234_5678_9abc_def0+block0():+ v0 = iconst.i32 10+<ss0, 2, 2> v1 = call fn0(v0, v0)+ return v1+}++; function %f1() -> i32 fast {+; ss0 = explicit_slot 64, key = 1311768467463790320+; ss1 = explicit_slot 64, key = 18364758544493064720+; sig0 = (i32, i32) -> i32 fast+; fn0 = %f0 sig0+;+; block0:+; v0 = iconst.i32 10+; jump block1+;+; block1:+; <ss0, 2, 2, ss1, 1, 0> sequence_point+; v3 = iadd.i32 v0, v0 ; v0 = 10, v0 = 10+; jump block2(v3)+;+; block2(v2: i32):+; v1 -> v2+; return v1+; }+diff --git a/cranelift/filetests/filetests/isa/x64/debug.clif b/cranelift/filetests/filetests/isa/x64/debug.clifnew file mode 100644index 0000000000..16fc15245e--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/debug.clif@@ -0,0 +1,108 @@+test compile precise-output+target x86_64++function %f0(i32, i32) -> i32 {+ ss0 = explicit_slot 64, key = 1234+ ;; not colocated -- shows interesting case where tags attach to+ ;; multiple VCode insts.+ fn0 = %f1(i32, i32)+block0(v0: i32, v1: i32):+<ss0, 0, 0> call fn0(v0, v1)+<ss0, 1, 0> sequence_point+ v2 = iadd v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; subq $0x50, %rsp+; movq %r12, 0x40(%rsp)+; movq %r14, 0x48(%rsp)+; block0:+; load_ext_name %f1+0, %rdx+; ; ^-- debug @ Pre: [StackSlot(ss0), User(0), User(0)]+; movq %rsi, %r12+; movq %rdi, %r14+; call *%rdx+; ; ^-- debug @ Post: [StackSlot(ss0), User(0), User(0)]+; sequence_point+; ; ^-- debug @ Pre: [StackSlot(ss0), User(1), User(0)]+; leal (%r14, %r12), %eax+; movq 0x40(%rsp), %r12+; movq 0x48(%rsp), %r14+; addq $0x50, %rsp+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; subq $0x50, %rsp+; movq %r12, 0x40(%rsp)+; movq %r14, 0x48(%rsp)+; block1: ; offset 0x12+; movabsq $0, %rdx ; reloc_external Abs8 %f1 0+; movq %rsi, %r12+; movq %rdi, %r14+; callq *%rdx+; leal (%r14, %r12), %eax+; movq 0x40(%rsp), %r12+; movq 0x48(%rsp), %r14+; addq $0x50, %rsp+; movq %rbp, %rsp+; popq %rbp+; retq++function %f1(i32, i32) -> i32 {+ ss0 = explicit_slot 64, key = 1234+ fn0 = colocated %f1(i32, i32)+block0(v0: i32, v1: i32):+<ss0, 0, 0> call fn0(v0, v1)+<ss0, 1, 0> sequence_point+ v2 = iadd v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; subq $0x50, %rsp+; movq %rbx, 0x40(%rsp)+; movq %r15, 0x48(%rsp)+; block0:+; movq %rdi, %rbx+; movq %rsi, %r15+; call TestCase(%f1)+; ; ^-- debug @ Post: [StackSlot(ss0), User(0), User(0)]+; sequence_point+; ; ^-- debug @ Pre: [StackSlot(ss0), User(1), User(0)]+; leal (%rbx, %r15), %eax+; movq 0x40(%rsp), %rbx+; movq 0x48(%rsp), %r15+; addq $0x50, %rsp+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; subq $0x50, %rsp+; movq %rbx, 0x40(%rsp)+; movq %r15, 0x48(%rsp)+; block1: ; offset 0x12+; movq %rdi, %rbx+; movq %rsi, %r15+; callq 0x1d ; reloc_external CallPCRel4 %f1 -4+; leal (%rbx, %r15), %eax+; movq 0x40(%rsp), %rbx+; movq 0x48(%rsp), %r15+; addq $0x50, %rsp+; movq %rbp, %rsp+; popq %rbp+; retq+diff --git a/cranelift/filetests/filetests/verifier/debug_tags.clif b/cranelift/filetests/filetests/verifier/debug_tags.clifnew file mode 100644index 0000000000..bb7fa77739--- /dev/null+++ b/cranelift/filetests/filetests/verifier/debug_tags.clif@@ -0,0 +1,7 @@+test verifier++function %f(i32, i32) -> i32 {+block0(v0: i32, v1: i32):+<0, 1> v2 = iadd.i32 v0, v1 ; error: debug tags present on non-call, non-sequence point instruction+ return v2+}\ No newline at end of filediff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rsindex ac4bee433d..8a7fad2323 100644--- a/cranelift/interpreter/src/step.rs+++ b/cranelift/interpreter/src/step.rs@@ -1321,6 +1321,8 @@ whereOpcode::TryCallIndirect => unimplemented!("TryCallIndirect"),Opcode::GetExceptionHandlerAddress => unimplemented!("GetExceptionHandlerAddress"),++ Opcode::SequencePoint => unimplemented!("SequencePoint"),})}diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rsindex 17b3f63c22..ce8feb2215 100644--- a/cranelift/reader/src/lexer.rs+++ b/cranelift/reader/src/lexer.rs@@ -19,6 +19,8 @@ pub enum Token<'a> {RBrace, // '}'LBracket, // '['RBracket, // ']'+ LAngle, // '<'+ RAngle, // '>'Minus, // '-'Plus, // '+'Multiply, // '*'@@ -482,6 +484,8 @@ impl<'a> Lexer<'a> {Some('}') => Some(self.scan_char(Token::RBrace)),Some('[') => Some(self.scan_char(Token::LBracket)),Some(']') => Some(self.scan_char(Token::RBracket)),+ Some('<') => Some(self.scan_char(Token::LAngle)),+ Some('>') => Some(self.scan_char(Token::RAngle)),Some(',') => Some(self.scan_char(Token::Comma)),Some('.') => Some(self.scan_char(Token::Dot)),Some(':') => Some(self.scan_char(Token::Colon)),diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rsindex 29faadee23..e2172ae99b 100644--- a/cranelift/reader/src/parser.rs+++ b/cranelift/reader/src/parser.rs@@ -15,8 +15,8 @@ use cranelift_codegen::ir::immediates::{};use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs};use cranelift_codegen::ir::pcc::{BaseExpr, Expr, Fact};-use cranelift_codegen::ir::types::*;-use cranelift_codegen::ir::{self, UserExternalNameRef};+use cranelift_codegen::ir::{self, StackSlotKey, UserExternalNameRef};+use cranelift_codegen::ir::{DebugTag, types::*};use cranelift_codegen::ir::{AbiParam, ArgumentExtension, ArgumentPurpose, Block, BlockArg, Constant, ConstantData,@@ -956,6 +956,38 @@ impl<'a> Parser<'a> {}}+ /// Parse an optional list of debug tags.+ fn optional_debug_tags(&mut self) -> ParseResult<Vec<DebugTag>> {+ if self.optional(Token::LAngle) {+ let mut tags = vec![];+ while !self.optional(Token::RAngle) {+ match self.token() {+ Some(Token::Integer(_)) => {+ let value: u32 = self.match_uimm32("expected a u32 value")?.into();+ tags.push(DebugTag::User(value));+ }+ Some(Token::StackSlot(slot)) => {+ self.consume();+ tags.push(DebugTag::StackSlot(StackSlot::from_u32(slot)));+ }+ _ => {+ return err!(+ self.loc,+ "expected integer user value or stack slot in debug tags"+ );+ }+ }+ if !self.optional(Token::Comma) {+ self.match_token(Token::RAngle, "expected `,` or `>`")?;+ break;+ }+ }+ Ok(tags)+ } else {+ Ok(vec![])+ }+ }+/// Parse a list of literals (i.e. integers, floats, booleans); e.g. `0 1 2 3`, usually as/// part of something like `vconst.i32x4 [0 1 2 3]`.fn parse_literals_to_constant_data(&mut self, ty: Type) -> ParseResult<ConstantData> {@@ -1510,7 +1542,7 @@ impl<'a> Parser<'a> {// | "spill_slot"// | "incoming_arg"// | "outgoing_arg"- // stack-slot-flag ::= "align" "=" Bytes+ // stack-slot-flag ::= "align" "=" Bytes | "key" "=" uimm64fn parse_stack_slot_decl(&mut self) -> ParseResult<(StackSlot, StackSlotData)> {let ss = self.match_ss("expected stack slot number: ss«n»")?;self.match_token(Token::Equal, "expected '=' in stack slot declaration")?;@@ -1527,29 +1559,42 @@ impl<'a> Parser<'a> {return err!(self.loc, "stack slot too large");}- // Parse flags.- let align = if self.token() == Some(Token::Comma) {+ let mut align = 1;+ let mut key = None;++ while self.token() == Some(Token::Comma) {self.consume();- self.match_token(- Token::Identifier("align"),- "expected a valid stack-slot flag (currently only `align`)",- )?;- self.match_token(Token::Equal, "expected `=` after flag")?;- let align: i64 = self- .match_imm64("expected alignment-size after `align` flag")?- .into();- u32::try_from(align)- .map_err(|_| self.error("alignment must be a 32-bit unsigned integer"))?- } else {- 1- };+ match self.token() {+ Some(Token::Identifier("align")) => {+ self.consume();+ self.match_token(Token::Equal, "expected `=` after flag")?;+ let align64: i64 = self+ .match_imm64("expected alignment-size after `align` flag")?+ .into();+ align = u32::try_from(align64)+ .map_err(|_| self.error("alignment must be a 32-bit unsigned integer"))?;+ }+ Some(Token::Identifier("key")) => {+ self.consume();+ self.match_token(Token::Equal, "expected `=` after flag")?;+ let value = self.match_uimm64("expected `u64` value for `key` flag")?;+ key = Some(StackSlotKey::new(value.into()));+ }+ _ => {+ return Err(self.error("invalid flag for stack slot"));+ }+ }+ }if !align.is_power_of_two() {return err!(self.loc, "stack slot alignment is not a power of two");}let align_shift = u8::try_from(align.ilog2()).unwrap(); // Always succeeds: range 0..=31.- let data = StackSlotData::new(kind, bytes as u32, align_shift);+ let data = match key {+ Some(key) => StackSlotData::new_with_key(kind, bytes as u32, align_shift, key),+ None => StackSlotData::new(kind, bytes as u32, align_shift),+ };// Collect any trailing comments.self.token();@@ -2103,11 +2148,14 @@ impl<'a> Parser<'a> {Some(Token::Value(_))| Some(Token::Identifier(_))| Some(Token::LBracket)- | Some(Token::SourceLoc(_)) => true,+ | Some(Token::SourceLoc(_))+ | Some(Token::LAngle) => true,_ => false,} {let srcloc = self.optional_srcloc()?;+ let debug_tags = self.optional_debug_tags()?;+// We need to parse instruction results here because they are shared// between the parsing of value aliases and the parsing of instructions.//@@ -2127,10 +2175,10 @@ impl<'a> Parser<'a> {}Some(Token::Equal) => {self.consume();- self.parse_instruction(&results, srcloc, ctx, block)?;+ self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?;}_ if !results.is_empty() => return err!(self.loc, "expected -> or ="),- _ => self.parse_instruction(&results, srcloc, ctx, block)?,+ _ => self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?,}}@@ -2521,6 +2569,7 @@ impl<'a> Parser<'a> {&mut self,results: &[Value],srcloc: ir::SourceLoc,+ debug_tags: Vec<DebugTag>,ctx: &mut Context,block: Block,) -> ParseResult<()> {@@ -2560,16 +2609,22 @@ impl<'a> Parser<'a> {// instruction ::= [inst-results "="] Opcode(opc) ["." Type] * ...let inst_data = self.parse_inst_operands(ctx, opcode, explicit_ctrl_type)?;- // We're done parsing the instruction data itself.- //- // We still need to check that the number of result values in the source- // matches the opcode or function call signature. We also need to create- // values with the right type for all the instruction results and parse- // and attach stack map entries, if present.let ctrl_typevar = self.infer_typevar(ctx, opcode, explicit_ctrl_type, &inst_data)?;let inst = ctx.function.dfg.make_inst(inst_data);- if opcode.is_call() && !opcode.is_return() && self.optional(Token::Comma) {- self.match_identifier("stack_map", "expected `stack_map = [...]`")?;++ // Attach stack map, if present.+ if self.optional(Token::Comma) {+ self.match_token(+ Token::Identifier("stack_map"),+ "expected `stack_map = [...]`",+ )?;+ if !opcode.is_call() || opcode.is_return() {+ return err!(+ self.loc,+ "stack map can only be attached to a (non-tail) call"+ );+ }+self.match_token(Token::Equal, "expected `= [...]`")?;self.match_token(Token::LBracket, "expected `[...]`")?;while !self.optional(Token::RBracket) {@@ -2594,6 +2649,13 @@ impl<'a> Parser<'a> {}}}++ // We're done parsing the instruction data itself.+ //+ // We still need to check that the number of result values in+ // the source matches the opcode or function call+ // signature. We also need to create values with the right+ // type for all the instruction results.let num_results =ctx.function.dfg@@ -2606,6 +2668,9 @@ impl<'a> Parser<'a> {if !srcloc.is_default() {ctx.function.set_srcloc(inst, srcloc);}+ if !debug_tags.is_empty() {+ ctx.function.debug_tags.set(inst, debug_tags);+ }if results.len() != num_results {return err!(