Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: add debug tag infrastructure. (#11768)

Chris Fallina3d6e407173d
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs
index 7480a4d44e..9604c6904b 100644
--- a/cranelift/codegen/meta/src/shared/instructions.rs
+++ b/cranelift/codegen/meta/src/shared/instructions.rs
@@ -3902,4 +3902,20 @@ pub(crate) fn define(
Operand::new("a", &TxN.dynamic_to_vector()).with_doc("New fixed vector"),
]),
);
+
+ ig.push(
+ Inst::new(
+ "sequence_point",
+ r#"
+ A compiler barrier that acts as an immovable marker from IR input to machine-code output.
+
+ This "sequence point" can have debug tags attached to it, and these tags will be
+ noted in the output `MachBuffer`.
+
+ It prevents motion of any other side-effects across this boundary.
+ "#,
+ &formats.nullary,
+ )
+ .other_side_effects(),
+ );
}
diff --git a/cranelift/codegen/src/inline.rs b/cranelift/codegen/src/inline.rs
index 9a425086fd..1ca82203f3 100644
--- a/cranelift/codegen/src/inline.rs
+++ b/cranelift/codegen/src/inline.rs
@@ -20,7 +20,7 @@
//! Cranelift the body of the callee that is to be inlined.
use crate::cursor::{Cursor as _, FuncCursor};
-use crate::ir::{self, ExceptionTableData, ExceptionTableItem, InstBuilder as _};
+use crate::ir::{self, DebugTag, ExceptionTableData, ExceptionTableItem, InstBuilder as _};
use crate::result::CodegenResult;
use crate::trace;
use crate::traversals::Dfs;
@@ -366,6 +366,13 @@ fn inline_one(
// callee.
let mut last_inlined_block = inline_block_layout(func, call_block, callee, &entity_map);
+ // Get a copy of debug tags on the call instruction; these are
+ // prepended to debug tags on inlined instructions. Remove them
+ // from the call itself as it will be rewritten to a jump (which
+ // cannot have tags).
+ let call_debug_tags = func.debug_tags.get(call_inst).to_vec();
+ func.debug_tags.set(call_inst, []);
+
// Translate each instruction from the callee into the caller,
// appending them to their associated block in the caller.
//
@@ -403,6 +410,29 @@ fn inline_one(
let inlined_inst = func.dfg.make_inst(inlined_inst_data);
func.layout.append_inst(inlined_inst, inlined_block);
+ // Copy over debug tags, translating referenced entities
+ // as appropriate.
+ let debug_tags = callee.debug_tags.get(callee_inst);
+ // If there are tags on the inlined instruction, we always
+ // add tags, and we prepend any tags from the call
+ // instruction; but we don't add tags if only the callsite
+ // had them (this would otherwise mean that every single
+ // instruction in an inlined function body would get
+ // tags).
+ if !debug_tags.is_empty() {
+ let tags = call_debug_tags
+ .iter()
+ .cloned()
+ .chain(debug_tags.iter().map(|tag| match *tag {
+ DebugTag::User(value) => DebugTag::User(value),
+ DebugTag::StackSlot(slot) => {
+ DebugTag::StackSlot(entity_map.inlined_stack_slot(slot))
+ }
+ }))
+ .collect::<SmallVec<[_; 4]>>();
+ func.debug_tags.set(inlined_inst, tags);
+ }
+
let opcode = callee.dfg.insts[callee_inst].opcode();
if opcode.is_return() {
// Instructions that return do not define any values, so we
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index cdaa5ebc25..7b42e17286 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -147,7 +147,8 @@ pub fn has_memory_fence_semantics(op: Opcode) -> bool {
| Opcode::AtomicLoad
| Opcode::AtomicStore
| Opcode::Fence
- | Opcode::Debugtrap => true,
+ | Opcode::Debugtrap
+ | Opcode::SequencePoint => true,
Opcode::Call | Opcode::CallIndirect | Opcode::TryCall | Opcode::TryCallIndirect => true,
op if op.can_trap() => true,
_ => false,
diff --git a/cranelift/codegen/src/ir/debug_tags.rs b/cranelift/codegen/src/ir/debug_tags.rs
new file mode 100644
index 0000000000..39d137c297
--- /dev/null
+++ b/cranelift/codegen/src/ir/debug_tags.rs
@@ -0,0 +1,141 @@
+//! Debug tag storage.
+//!
+//! Cranelift permits the embedder to place "debug tags" on
+//! instructions in CLIF. These tags are sequences of items of various
+//! kinds, with no other meaning imposed by Cranelift. They are passed
+//! through to metadata provided alongside the compilation result.
+//!
+//! When Cranelift inlines a function, it will prepend any tags from
+//! the call instruction at the inlining callsite to tags on all
+//! inlined instructions.
+//!
+//! These tags can be used, for example, to identify stackslots that
+//! store user state, or to denote positions in user source. In
+//! general, the intent is to allow perfect reconstruction of original
+//! (source-level) program state in an instrumentation-based
+//! debug-info scheme, as long as the instruction(s) on which these
+//! tags are attached are preserved. This will be the case for any
+//! instructions with side-effects.
+//!
+//! A few answers to design questions that lead to this design:
+//!
+//! - Why not use the SourceLoc mechanism? Debug tags are richer than
+//! that infrastructure because they preserve inlining location and
+//! are interleaved properly with any other tags describing the
+//! frame.
+//! - Why not attach debug tags only to special sequence-point
+//! instructions? This is driven by inlining: we should have the
+//! semantic information about a callsite attached directly to the
+//! call and observe it there, not have a magic "look backward to
+//! find a sequence point" behavior in the inliner.
+//!
+//! In other words, the needs of preserving "virtual" frames across an
+//! inlining transform drive this design.
+
+use crate::ir::{Inst, StackSlot};
+use alloc::collections::BTreeMap;
+use alloc::vec::Vec;
+use core::ops::Range;
+
+/// Debug tags for instructions.
+#[derive(Clone, PartialEq, Hash, Default)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub struct DebugTags {
+ /// Pool of tags, referred to by `insts` below.
+ tags: Vec<DebugTag>,
+
+ /// Per-instruction range for its list of tags in the tag pool (if
+ /// any).
+ ///
+ /// Note: we don't use `PackedOption` and `EntityList` here
+ /// because the values that we are storing are not entities.
+ insts: BTreeMap<Inst, Range<u32>>,
+}
+
+/// One debug tag.
+#[derive(Clone, Debug, PartialEq, Hash)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub enum DebugTag {
+ /// User-specified `u32` value, opaque to Cranelift.
+ User(u32),
+
+ /// A stack slot reference.
+ StackSlot(StackSlot),
+}
+
+impl DebugTags {
+ /// Set the tags on an instruction, overwriting existing tag list.
+ ///
+ /// Tags can only be set on call instructions (those for which
+ /// [`crate::Opcode::is_call()`] returns `true`) and on
+ /// `sequence_point` instructions. This property is checked by the
+ /// CLIF verifier.
+ pub fn set(&mut self, inst: Inst, tags: impl IntoIterator<Item = DebugTag>) {
+ let start = u32::try_from(self.tags.len()).unwrap();
+ self.tags.extend(tags);
+ let end = u32::try_from(self.tags.len()).unwrap();
+ if end > start {
+ self.insts.insert(inst, start..end);
+ } else {
+ self.insts.remove(&inst);
+ }
+ }
+
+ /// Get the tags associated with an instruction.
+ pub fn get(&self, inst: Inst) -> &[DebugTag] {
+ if let Some(range) = self.insts.get(&inst) {
+ let start = usize::try_from(range.start).unwrap();
+ let end = usize::try_from(range.end).unwrap();
+ &self.tags[start..end]
+ } else {
+ &[]
+ }
+ }
+
+ /// Does the given instruction have any tags?
+ pub fn has(&self, inst: Inst) -> bool {
+ // We rely on the invariant that an entry in the map is
+ // present only if the list range is non-empty.
+ self.insts.contains_key(&inst)
+ }
+
+ /// Clone the tags from one instruction to another.
+ ///
+ /// This clone is cheap (references the same underlying storage)
+ /// because the tag lists are immutable.
+ pub fn clone_tags(&mut self, from: Inst, to: Inst) {
+ if let Some(range) = self.insts.get(&from).cloned() {
+ self.insts.insert(to, range);
+ } else {
+ self.insts.remove(&to);
+ }
+ }
+
+ /// Are any debug tags present?
+ ///
+ /// This is used for adjusting margins when pretty-printing CLIF.
+ pub fn is_empty(&self) -> bool {
+ self.insts.is_empty()
+ }
+
+ /// Clear all tags.
+ pub fn clear(&mut self) {
+ self.insts.clear();
+ self.tags.clear();
+ }
+}
+
+impl core::fmt::Display for DebugTag {
+ fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+ match self {
+ DebugTag::User(value) => write!(f, "{value}"),
+ DebugTag::StackSlot(slot) => write!(f, "{slot}"),
+ }
+ }
+}
diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs
index 02f2a31cc7..80926edcdf 100644
--- a/cranelift/codegen/src/ir/function.rs
+++ b/cranelift/codegen/src/ir/function.rs
@@ -5,6 +5,7 @@
use crate::HashMap;
use crate::entity::{PrimaryMap, SecondaryMap};
+use crate::ir::DebugTags;
use crate::ir::{
self, Block, DataFlowGraph, DynamicStackSlot, DynamicStackSlotData, DynamicStackSlots,
DynamicType, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Inst, JumpTable,
@@ -190,6 +191,22 @@ pub struct FunctionStencil {
/// interpreted by Cranelift, only preserved.
pub srclocs: SourceLocs,
+ /// Opaque debug-info tags on sequence-point and call
+ /// instructions.
+ ///
+ /// These tags are not interpreted by Cranelift, and are passed
+ /// through to compilation-result metadata. The only semantic
+ /// structure that Cranelift imposes is that when inlining, it
+ /// prepends the callsite call instruction's tags to the tags on
+ /// inlined instructions.
+ ///
+ /// In order to ensure clarity around guaranteed compiler
+ /// behavior, tags are only permitted on instructions whose
+ /// presence and sequence will remain the same in the compiled
+ /// output: namely, `sequence_point` instructions and ordinary
+ /// call instructions.
+ pub debug_tags: DebugTags,
+
/// An optional global value which represents an expression evaluating to
/// the stack limit for this function. This `GlobalValue` will be
/// interpreted in the prologue, if necessary, to insert a stack check to
@@ -209,6 +226,7 @@ impl FunctionStencil {
self.dfg.clear();
self.layout.clear();
self.srclocs.clear();
+ self.debug_tags.clear();
self.stack_limit = None;
}
@@ -408,6 +426,7 @@ impl Function {
layout: Layout::new(),
srclocs: SecondaryMap::new(),
stack_limit: None,
+ debug_tags: DebugTags::default(),
},
params: FunctionParameters::new(),
}
diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs
index 4519d5d74e..eca17e7bad 100644
--- a/cranelift/codegen/src/ir/mod.rs
+++ b/cranelift/codegen/src/ir/mod.rs
@@ -4,6 +4,7 @@ mod atomic_rmw_op;
mod builder;
pub mod condcodes;
pub mod constant;
+mod debug_tags;
pub mod dfg;
pub mod dynamic_type;
pub mod entities;
@@ -36,6 +37,7 @@ pub use crate::ir::builder::{
InsertBuilder, InstBuilder, InstBuilderBase, InstInserterBase, ReplaceBuilder,
};
pub use crate::ir::constant::{ConstantData, ConstantPool};
+pub use crate::ir::debug_tags::{DebugTag, DebugTags};
pub use crate::ir::dfg::{BlockData, DataFlowGraph, ValueDef};
pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes, dynamic_to_fixed};
pub use crate::ir::entities::{
@@ -64,7 +66,7 @@ pub use crate::ir::progpoint::ProgramPoint;
pub use crate::ir::sourceloc::RelSourceLoc;
pub use crate::ir::sourceloc::SourceLoc;
pub use crate::ir::stackslot::{
- DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKind, StackSlots,
+ DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKey, StackSlotKind, StackSlots,
};
pub use crate::ir::trapcode::TrapCode;
pub use crate::ir::types::Type;
diff --git a/cranelift/codegen/src/ir/stackslot.rs b/cranelift/codegen/src/ir/stackslot.rs
index cfcb8b2c9c..1ad2a5d7d8 100644
--- a/cranelift/codegen/src/ir/stackslot.rs
+++ b/cranelift/codegen/src/ir/stackslot.rs
@@ -69,6 +69,37 @@ pub struct StackSlotData {
/// be aligned according to other considerations, such as minimum
/// stack slot size or machine word size, as well.
pub align_shift: u8,
+
+ /// Opaque stackslot metadata handle, passed through to
+ /// compilation result metadata describing stackslot location.
+ ///
+ /// In the face of compiler transforms like inlining that may move
+ /// stackslots between functions, when an embedder wants to
+ /// externally observe stackslots, it needs a first-class way for
+ /// the identity of stackslots to be carried along with the IR
+ /// entities. This opaque `StackSlotKey` allows the embedder to do
+ /// so.
+ pub key: Option<StackSlotKey>,
+}
+
+/// An opaque key uniquely identifying a stack slot.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub struct StackSlotKey(u64);
+impl StackSlotKey {
+ /// Construct a [`StackSlotKey`] from raw bits.
+ ///
+ /// An embedder can use any 64-bit value to describe a stack slot;
+ /// there are no restrictions, and the value does not mean
+ /// anything to Cranelift itself.
+ pub fn new(value: u64) -> StackSlotKey {
+ StackSlotKey(value)
+ }
+
+ /// Get the raw bits from the [`StackSlotKey`].
+ pub fn bits(&self) -> u64 {
+ self.0
+ }
}
impl StackSlotData {
@@ -78,23 +109,40 @@ impl StackSlotData {
kind,
size,
align_shift,
+ key: None,
+ }
+ }
+
+ /// Create a stack slot with the specified byte size and alignment
+ /// and the given user-defined key.
+ pub fn new_with_key(
+ kind: StackSlotKind,
+ size: StackSize,
+ align_shift: u8,
+ key: StackSlotKey,
+ ) -> Self {
+ Self {
+ kind,
+ size,
+ align_shift,
+ key: Some(key),
}
}
}
impl fmt::Display for StackSlotData {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- if self.align_shift != 0 {
- write!(
- f,
- "{} {}, align = {}",
- self.kind,
- self.size,
- 1u32 << self.align_shift
- )
+ let align_shift = if self.align_shift != 0 {
+ format!(", align = {}", 1u32 << self.align_shift)
} else {
- write!(f, "{} {}", self.kind, self.size)
- }
+ "".into()
+ };
+ let key = match self.key {
+ Some(value) => format!(", key = {}", value.bits()),
+ None => "".into(),
+ };
+
+ write!(f, "{} {}{align_shift}{key}", self.kind, self.size)
}
}
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 9251f4d8b1..2d1eb8f1b2 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -997,6 +997,9 @@
(LabelAddress (dst WritableReg)
(label MachLabel))
+ ;; A pseudoinstruction that serves as a sequence point.
+ (SequencePoint)
+
;; Emits an inline stack probe loop.
;;
;; Note that this is emitted post-regalloc so `start` and `end` can be
@@ -5203,3 +5206,8 @@
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LabelAddress dst label))))
dst))
+
+;; Helper for creating a `SequencePoint` instruction.
+(decl a64_sequence_point () SideEffectNoResult)
+(rule (a64_sequence_point)
+ (SideEffectNoResult.Inst (MInst.SequencePoint)))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index db910a650a..91beb84efb 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -3547,6 +3547,10 @@ impl MachInstEmit for Inst {
sink.use_label_at_offset(offset, label, LabelUse::Adr21);
}
+ &Inst::SequencePoint { .. } => {
+ // Nothing.
+ }
+
&Inst::StackProbeLoop { start, end, step } => {
assert!(emit_info.0.enable_probestack());
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index dbff1df1b6..1ba8ec50ff 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -919,6 +919,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
Inst::LabelAddress { dst, .. } => {
collector.reg_def(dst);
}
+ Inst::SequencePoint { .. } => {}
Inst::StackProbeLoop { start, end, .. } => {
collector.reg_early_def(start);
collector.reg_use(end);
@@ -2893,6 +2894,9 @@ impl Inst {
let dst = pretty_print_reg(dst.to_reg());
format!("label_address {dst}, {label:?}")
}
+ &Inst::SequencePoint {} => {
+ format!("sequence_point")
+ }
&Inst::StackProbeLoop { start, end, step } => {
let start = pretty_print_reg(start.to_reg());
let end = pretty_print_reg(end);
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index f304e5c620..c32edb2b1f 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -3271,3 +3271,9 @@
(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
(let ((succ_label MachLabel (block_exn_successor_label block idx)))
(a64_label_address succ_label)))
+
+;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (sequence_point))
+ (side_effect
+ (a64_sequence_point)))
diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs
index 98f4d5c7d6..892911edd8 100644
--- a/cranelift/codegen/src/isa/aarch64/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/mod.rs
@@ -74,11 +74,8 @@ impl TargetIsa for AArch64Backend {
let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;
let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
- let frame_size = emit_result.frame_size;
let value_labels_ranges = emit_result.value_labels_ranges;
let buffer = emit_result.buffer;
- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;
- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;
if let Some(disasm) = emit_result.disasm.as_ref() {
log::debug!("disassembly:\n{disasm}");
@@ -86,11 +83,8 @@ impl TargetIsa for AArch64Backend {
Ok(CompiledCodeStencil {
buffer,
- frame_size,
vcode: emit_result.disasm,
value_labels_ranges,
- sized_stackslot_offsets,
- dynamic_stackslot_offsets,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
})
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle
index 4b008a8867..901753841c 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle
+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle
@@ -95,6 +95,9 @@
;; A pseudoinstruction that loads the address of a label.
(LabelAddress (dst WritableXReg)
(label MachLabel))
+
+ ;; A pseudoinstruction that serves as a sequence point.
+ (SequencePoint)
)
)
@@ -792,3 +795,8 @@
(let ((dst WritableReg (temp_writable_xreg))
(_ Unit (emit (MInst.LabelAddress dst label))))
dst))
+
+;; Helper for creating a `SequencePoint` instruction.
+(decl pulley_sequence_point () SideEffectNoResult)
+(rule (pulley_sequence_point)
+ (SideEffectNoResult.Inst (MInst.SequencePoint)))
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
index 08d8b5eaf7..c8de063259 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
@@ -596,6 +596,10 @@ fn pulley_emit<P>(
let end = sink.cur_offset();
sink.use_label_at_offset(end - 4, *label, LabelUse::PcRel);
}
+
+ Inst::SequencePoint { .. } => {
+ // Nothing.
+ }
}
}
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
index bc2490f7bc..606ec93ae3 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
@@ -335,6 +335,8 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
Inst::LabelAddress { dst, label: _ } => {
collector.reg_def(dst);
}
+
+ Inst::SequencePoint { .. } => {}
}
}
@@ -834,6 +836,10 @@ impl Inst {
let dst = format_reg(dst.to_reg().to_reg());
format!("label_address {dst}, {label:?}")
}
+
+ Inst::SequencePoint {} => {
+ format!("sequence_point")
+ }
}
}
}
diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle
index 12770d844e..7cde5f2027 100644
--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle
+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle
@@ -1836,3 +1836,9 @@
(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
(let ((succ_label MachLabel (block_exn_successor_label block idx)))
(pulley_label_address succ_label)))
+
+;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (sequence_point))
+ (side_effect
+ (pulley_sequence_point)))
diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rs
index 0b2cf027f0..c3f16efa2b 100644
--- a/cranelift/codegen/src/isa/pulley_shared/mod.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs
@@ -180,11 +180,8 @@ where
let want_disasm =
want_disasm || (cfg!(feature = "trace-log") && log::log_enabled!(log::Level::Debug));
let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
- let frame_size = emit_result.frame_size;
let value_labels_ranges = emit_result.value_labels_ranges;
let buffer = emit_result.buffer;
- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;
- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;
if let Some(disasm) = emit_result.disasm.as_ref() {
log::debug!("disassembly:\n{disasm}");
@@ -192,11 +189,8 @@ where
Ok(CompiledCodeStencil {
buffer,
- frame_size,
vcode: emit_result.disasm,
value_labels_ranges,
- sized_stackslot_offsets,
- dynamic_stackslot_offsets,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
})
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index f118fcf710..62774b59fe 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -360,6 +360,9 @@
(EmitIsland
;; The needed space before the next deadline.
(needed_space u32))
+
+ ;; A pseudoinstruction that serves as a sequence point.
+ (SequencePoint)
))
(type AtomicOP (enum
@@ -3262,3 +3265,8 @@
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LabelAddress dst label))))
dst))
+
+;; Helper for creating a `SequencePoint` instruction.
+(decl rv64_sequence_point () SideEffectNoResult)
+(rule (rv64_sequence_point)
+ (SideEffectNoResult.Inst (MInst.SequencePoint)))
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index e7a64b605f..77bd801c4f 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -195,6 +195,7 @@ impl Inst {
| Inst::Unwind { .. }
| Inst::DummyUse { .. }
| Inst::LabelAddress { .. }
+ | Inst::SequencePoint { .. }
| Inst::Popcnt { .. }
| Inst::Cltz { .. }
| Inst::Brev8 { .. }
@@ -2769,6 +2770,10 @@ impl Inst {
sink.bind_label(jump_around_label, &mut state.ctrl_plane);
}
}
+
+ Inst::SequencePoint { .. } => {
+ // Nothing.
+ }
}
}
}
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
index 5f4c037b29..4a3890b093 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -699,6 +699,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
Inst::LabelAddress { dst, .. } => {
collector.reg_def(dst);
}
+ Inst::SequencePoint { .. } => {}
}
}
@@ -1680,6 +1681,10 @@ impl Inst {
let dst = format_reg(dst.to_reg());
format!("label_address {dst}, {label:?}")
}
+
+ Inst::SequencePoint {} => {
+ format!("sequence_point")
+ }
}
}
}
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index d2329133b7..920e5c2119 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -3138,3 +3138,9 @@
(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
(let ((succ_label MachLabel (block_exn_successor_label block idx)))
(rv64_label_address succ_label)))
+
+;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (sequence_point))
+ (side_effect
+ (rv64_sequence_point)))
diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs
index 3ae2b6dfb3..873b698510 100644
--- a/cranelift/codegen/src/isa/riscv64/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/mod.rs
@@ -75,11 +75,8 @@ impl TargetIsa for Riscv64Backend {
let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug);
let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
- let frame_size = emit_result.frame_size;
let value_labels_ranges = emit_result.value_labels_ranges;
let buffer = emit_result.buffer;
- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;
- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;
if let Some(disasm) = emit_result.disasm.as_ref() {
log::debug!("disassembly:\n{disasm}");
@@ -87,11 +84,8 @@ impl TargetIsa for Riscv64Backend {
Ok(CompiledCodeStencil {
buffer,
- frame_size,
vcode: emit_result.disasm,
value_labels_ranges,
- sized_stackslot_offsets,
- dynamic_stackslot_offsets,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
})
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 91d0227b3c..46576a42d4 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -1019,6 +1019,9 @@
;; A pseudoinstruction that loads the address of a label.
(LabelAddress (dst WritableReg)
(label MachLabel))
+
+ ;; A pseudoinstruction that serves as a sequence point.
+ (SequencePoint)
))
;; Primitive types used in instruction formats.
@@ -4941,6 +4944,11 @@
(_ Unit (emit (MInst.LabelAddress dst label))))
dst))
+;; Helper for creating a `SequencePoint` instruction.
+(decl s390x_sequence_point () SideEffectNoResult)
+(rule (s390x_sequence_point)
+ (SideEffectNoResult.Inst (MInst.SequencePoint)))
+
;; Implicit conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(convert WritableRegPair RegPair writable_regpair_to_regpair)
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index 98718d1ee1..d67bd5f56c 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -3544,6 +3544,10 @@ impl Inst {
};
inst.emit(sink, emit_info, state);
}
+
+ &Inst::SequencePoint { .. } => {
+ // Nothing.
+ }
}
state.clear_post_insn();
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 256de41504..8c16117ec6 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -284,6 +284,8 @@ impl Inst {
Inst::DummyUse { .. } => InstructionSet::Base,
Inst::LabelAddress { .. } => InstructionSet::Base,
+
+ Inst::SequencePoint { .. } => InstructionSet::Base,
}
}
@@ -1011,6 +1013,7 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Ope
Inst::LabelAddress { dst, .. } => {
collector.reg_def(dst);
}
+ Inst::SequencePoint { .. } => {}
}
}
@@ -3414,6 +3417,9 @@ impl Inst {
let dst = pretty_print_reg(dst.to_reg());
format!("label_address {dst}, {label:?}")
}
+ &Inst::SequencePoint {} => {
+ format!("sequence_point")
+ }
}
}
}
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index b86358749c..41d8507f6d 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -4073,3 +4073,9 @@
(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
(let ((succ_label MachLabel (block_exn_successor_label block idx)))
(s390x_label_address succ_label)))
+
+;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (sequence_point))
+ (side_effect
+ (s390x_sequence_point)))
diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs
index 0434c4fcba..8768b4eebf 100644
--- a/cranelift/codegen/src/isa/s390x/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/mod.rs
@@ -74,11 +74,8 @@ impl TargetIsa for S390xBackend {
let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;
let emit_result = vcode.emit(&regalloc_result, want_disasm, flags, ctrl_plane);
- let frame_size = emit_result.frame_size;
let value_labels_ranges = emit_result.value_labels_ranges;
let buffer = emit_result.buffer;
- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;
- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;
if let Some(disasm) = emit_result.disasm.as_ref() {
log::debug!("disassembly:\n{disasm}");
@@ -86,11 +83,8 @@ impl TargetIsa for S390xBackend {
Ok(CompiledCodeStencil {
buffer,
- frame_size,
vcode: emit_result.disasm,
value_labels_ranges,
- sized_stackslot_offsets,
- dynamic_stackslot_offsets,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
})
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index c8ea499e6e..c5c1c7c891 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -320,6 +320,9 @@
(LabelAddress (dst WritableGpr)
(label MachLabel))
+ ;; A pseudoinstruction that serves as a sequence point.
+ (SequencePoint)
+
;; An instruction assembled outside of cranelift-codegen.
(External (inst AssemblerInst))))
@@ -4097,3 +4100,8 @@
(decl libcall_3 (LibCall Reg Reg Reg) Reg)
(extern constructor libcall_3 libcall_3)
+
+;; Helper for creating a `SequencePoint` instruction.
+(decl x64_sequence_point () SideEffectNoResult)
+(rule (x64_sequence_point)
+ (SideEffectNoResult.Inst (MInst.SequencePoint)))
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 3f1582ce8d..a922cf7377 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1815,6 +1815,10 @@ pub(crate) fn emit(
asm::inst::leaq_rm::new(*dst, Amode::rip_relative(*label)).emit(sink, info, state);
}
+ Inst::SequencePoint { .. } => {
+ // Nothing.
+ }
+
Inst::External { inst } => {
let frame = state.frame_layout();
emit_maybe_shrink(
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index f01dd8781b..c3ee37a958 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -106,7 +106,8 @@ impl Inst {
| Inst::CoffTlsGetAddr { .. }
| Inst::Unwind { .. }
| Inst::DummyUse { .. }
- | Inst::LabelAddress { .. } => true,
+ | Inst::LabelAddress { .. }
+ | Inst::SequencePoint => true,
Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => emit_info.cmpxchg16b(),
@@ -828,6 +829,10 @@ impl PrettyPrint for Inst {
format!("label_address {dst}, {label:?}")
}
+ Inst::SequencePoint {} => {
+ format!("sequence_point")
+ }
+
Inst::External { inst } => {
format!("{inst}")
}
@@ -1193,6 +1198,8 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_def(dst);
}
+ Inst::SequencePoint { .. } => {}
+
Inst::External { inst } => {
inst.visit(&mut external::RegallocVisitor { collector });
}
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 1a051f41fd..ebfd43bbf1 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -5059,3 +5059,9 @@
(rule (lower (nop))
(invalid_reg))
+
+;; Rules for `sequence_point` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (sequence_point))
+ (side_effect
+ (x64_sequence_point)))
diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs
index f40fef58c0..0edaf5ccb7 100644
--- a/cranelift/codegen/src/isa/x64/mod.rs
+++ b/cranelift/codegen/src/isa/x64/mod.rs
@@ -83,11 +83,8 @@ impl TargetIsa for X64Backend {
let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;
let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
- let frame_size = emit_result.frame_size;
let value_labels_ranges = emit_result.value_labels_ranges;
let buffer = emit_result.buffer;
- let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;
- let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;
if let Some(disasm) = emit_result.disasm.as_ref() {
crate::trace!("disassembly:\n{}", disasm);
@@ -95,11 +92,8 @@ impl TargetIsa for X64Backend {
Ok(CompiledCodeStencil {
buffer,
- frame_size,
vcode: emit_result.disasm,
value_labels_ranges,
- sized_stackslot_offsets,
- dynamic_stackslot_offsets,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
})
diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs
index 75a0c8e3d2..09b356d699 100644
--- a/cranelift/codegen/src/isa/x64/pcc.rs
+++ b/cranelift/codegen/src/isa/x64/pcc.rs
@@ -211,6 +211,8 @@ pub(crate) fn check(
Inst::LabelAddress { .. } => Err(PccError::UnimplementedInst),
+ Inst::SequencePoint { .. } => Ok(()),
+
Inst::External { .. } => Ok(()), // TODO: unsure what to do about this!
}
}
diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs
index 64d37df76a..425edb25e1 100644
--- a/cranelift/codegen/src/lib.rs
+++ b/cranelift/codegen/src/lib.rs
@@ -68,9 +68,10 @@ pub use crate::machinst::buffer::{
OpenPatchRegion, PatchRegion,
};
pub use crate::machinst::{
- CallInfo, CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit,
- MachInstEmitState, MachLabel, RealReg, Reg, RelocDistance, TextSectionBuilder, VCodeConstant,
- VCodeConstantData, VCodeConstants, VCodeInst, Writable,
+ CallInfo, CompiledCode, Final, MachBuffer, MachBufferDebugTagList, MachBufferFinalized,
+ MachBufferFrameLayout, MachDebugTagPos, MachInst, MachInstEmit, MachInstEmitState, MachLabel,
+ RealReg, Reg, RelocDistance, TextSectionBuilder, VCodeConstant, VCodeConstantData,
+ VCodeConstants, VCodeInst, Writable,
};
mod alias_analysis;
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index cc13a6d49d..91510e5a46 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -100,8 +100,8 @@
use crate::CodegenError;
use crate::entity::SecondaryMap;
-use crate::ir::types::*;
use crate::ir::{ArgumentExtension, ArgumentPurpose, ExceptionTag, Signature};
+use crate::ir::{StackSlotKey, types::*};
use crate::isa::TargetIsa;
use crate::settings::ProbestackStrategy;
use crate::{ir, isa};
@@ -1142,6 +1142,8 @@ pub struct Callee<M: ABIMachineSpec> {
dynamic_stackslots: PrimaryMap<DynamicStackSlot, u32>,
/// Offsets to each sized stackslot.
sized_stackslots: PrimaryMap<StackSlot, u32>,
+ /// Descriptors for sized stackslots.
+ sized_stackslot_keys: SecondaryMap<StackSlot, Option<StackSlotKey>>,
/// Total stack size of all stackslots
stackslots_size: u32,
/// Stack size to be reserved for outgoing arguments.
@@ -1227,6 +1229,7 @@ impl<M: ABIMachineSpec> Callee<M> {
// Compute sized stackslot locations and total stackslot size.
let mut end_offset: u32 = 0;
let mut sized_stackslots = PrimaryMap::new();
+ let mut sized_stackslot_keys = SecondaryMap::new();
for (stackslot, data) in f.sized_stack_slots.iter() {
// We start our computation possibly unaligned where the previous
@@ -1250,6 +1253,7 @@ impl<M: ABIMachineSpec> Callee<M> {
debug_assert_eq!(stackslot.as_u32() as usize, sized_stackslots.len());
sized_stackslots.push(start_offset);
+ sized_stackslot_keys[stackslot] = data.key;
}
// Compute dynamic stackslot locations and total stackslot size.
@@ -1304,6 +1308,7 @@ impl<M: ABIMachineSpec> Callee<M> {
dynamic_stackslots,
dynamic_type_sizes,
sized_stackslots,
+ sized_stackslot_keys,
stackslots_size,
outgoing_args_size: 0,
tail_args_size,
@@ -2324,22 +2329,29 @@ impl<M: ABIMachineSpec> Callee<M> {
.expect("frame layout not computed before prologue generation")
}
- /// Returns the full frame size for the given function, after prologue
- /// emission has run. This comprises the spill slots and stack-storage
- /// slots as well as storage for clobbered callee-save registers, but
- /// not arguments arguments pushed at callsites within this function,
- /// or other ephemeral pushes.
- pub fn frame_size(&self) -> u32 {
+ /// Returns the offset from SP to FP for the given function, after
+ /// the prologue has set up the frame. This comprises the spill
+ /// slots and stack-storage slots as well as storage for clobbered
+ /// callee-save registers and outgoing arguments at callsites
+ /// (space for which is reserved during frame setup).
+ pub fn sp_to_fp_offset(&self) -> u32 {
let frame_layout = self.frame_layout();
- frame_layout.clobber_size + frame_layout.fixed_frame_storage_size
+ frame_layout.clobber_size
+ + frame_layout.fixed_frame_storage_size
+ + frame_layout.outgoing_args_size
}
/// Returns offset from the slot base in the current frame to the caller's SP.
pub fn slot_base_to_caller_sp_offset(&self) -> u32 {
+ // Note: this looks very similar to `frame_size()` above, but
+ // it differs in both endpoints: it measures from the bottom
+ // of stackslots, excluding outgoing args; and it includes the
+ // setup area (FP/LR) size and any extra tail-args space.
let frame_layout = self.frame_layout();
frame_layout.clobber_size
+ frame_layout.fixed_frame_storage_size
+ frame_layout.setup_area_size
+ + (frame_layout.tail_args_size - frame_layout.incoming_args_size)
}
/// Returns the size of arguments expected on the stack.
@@ -2390,6 +2402,28 @@ impl<M: ABIMachineSpec> Callee<M> {
let from = StackAMode::Slot(sp_off);
<M>::gen_load_stack(from, to_reg.map(Reg::from), ty)
}
+
+ /// Provide metadata to be emitted alongside machine code.
+ ///
+ /// This metadata describes the frame layout sufficiently to find
+ /// stack slots, so that runtimes and unwinders can observe state
+ /// set up by compiled code in stackslots allocated for that
+ /// purpose.
+ pub fn frame_slot_metadata(&self) -> MachBufferFrameLayout {
+ let frame_to_fp_offset = self.sp_to_fp_offset();
+ let mut stackslots = SecondaryMap::with_capacity(self.sized_stackslots.len());
+ let storage_area_base = self.frame_layout().outgoing_args_size;
+ for (slot, storage_area_offset) in &self.sized_stackslots {
+ stackslots[slot] = MachBufferStackSlot {
+ offset: storage_area_base.checked_add(*storage_area_offset).unwrap(),
+ key: self.sized_stackslot_keys[slot],
+ };
+ }
+ MachBufferFrameLayout {
+ frame_to_fp_offset,
+ stackslots,
+ }
+ }
}
/// An input argument to a call instruction: the vreg that is used,
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index b489b7c079..444ae2a25b 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -172,7 +172,7 @@
use crate::binemit::{Addend, CodeOffset, Reloc};
use crate::ir::function::FunctionParameters;
-use crate::ir::{ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode};
+use crate::ir::{DebugTag, ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode};
use crate::isa::unwind::UnwindInst;
use crate::machinst::{
BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst,
@@ -182,7 +182,7 @@ use crate::{MachInstEmitState, ir};
use crate::{VCodeConstantData, timing};
use core::ops::Range;
use cranelift_control::ControlPlane;
-use cranelift_entity::{PrimaryMap, entity_impl};
+use cranelift_entity::{PrimaryMap, SecondaryMap, entity_impl};
use smallvec::SmallVec;
use std::cmp::Ordering;
use std::collections::BinaryHeap;
@@ -254,6 +254,10 @@ pub struct MachBuffer<I: VCodeInst> {
exception_handlers: SmallVec<[MachExceptionHandler; 16]>,
/// Any source location mappings referring to this code.
srclocs: SmallVec<[MachSrcLoc<Stencil>; 64]>,
+ /// Any debug tags referring to this code.
+ debug_tags: Vec<MachDebugTags>,
+ /// Pool of debug tags referenced by `MachDebugTags` entries.
+ debug_tag_pool: Vec<DebugTag>,
/// Any user stack maps for this code.
///
/// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted
@@ -322,6 +326,10 @@ pub struct MachBuffer<I: VCodeInst> {
/// Indicates when a patchable region is currently open, to guard that it's
/// not possible to nest patchable regions.
open_patchable: bool,
+ /// Stack frame layout metadata. If provided for a MachBuffer
+ /// containing a function body, this allows interpretation of
+ /// runtime state given a view of an active stack frame.
+ frame_layout: Option<MachBufferFrameLayout>,
}
impl MachBufferFinalized<Stencil> {
@@ -338,9 +346,12 @@ impl MachBufferFinalized<Stencil> {
.into_iter()
.map(|srcloc| srcloc.apply_base_srcloc(base_srcloc))
.collect(),
+ debug_tags: self.debug_tags,
+ debug_tag_pool: self.debug_tag_pool,
user_stack_maps: self.user_stack_maps,
unwind_info: self.unwind_info,
alignment: self.alignment,
+ frame_layout: self.frame_layout,
}
}
}
@@ -367,11 +378,19 @@ pub struct MachBufferFinalized<T: CompilePhase> {
pub(crate) exception_handlers: SmallVec<[FinalizedMachExceptionHandler; 16]>,
/// Any source location mappings referring to this code.
pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>,
+ /// Any debug tags referring to this code.
+ pub(crate) debug_tags: Vec<MachDebugTags>,
+ /// Pool of debug tags referenced by `MachDebugTags` entries.
+ pub(crate) debug_tag_pool: Vec<DebugTag>,
/// Any user stack maps for this code.
///
/// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted
/// by code offset, and each stack map covers `span` bytes on the stack.
pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,
+ /// Stack frame layout metadata. If provided for a MachBuffer
+ /// containing a function body, this allows interpretation of
+ /// runtime state given a view of an active stack frame.
+ pub(crate) frame_layout: Option<MachBufferFrameLayout>,
/// Any unwind info at a given location.
pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,
/// The required alignment of this buffer.
@@ -447,6 +466,8 @@ impl<I: VCodeInst> MachBuffer<I> {
call_sites: SmallVec::new(),
exception_handlers: SmallVec::new(),
srclocs: SmallVec::new(),
+ debug_tags: vec![],
+ debug_tag_pool: vec![],
user_stack_maps: SmallVec::new(),
unwind_info: SmallVec::new(),
cur_srcloc: None,
@@ -464,6 +485,7 @@ impl<I: VCodeInst> MachBuffer<I> {
constants: Default::default(),
used_constants: Default::default(),
open_patchable: false,
+ frame_layout: None,
}
}
@@ -838,6 +860,8 @@ impl<I: VCodeInst> MachBuffer<I> {
// (end of buffer)
self.data.truncate(b.start as usize);
self.pending_fixup_records.truncate(b.fixup);
+
+ // Trim srclocs and debug tags now past the end of the buffer.
while let Some(last_srcloc) = self.srclocs.last_mut() {
if last_srcloc.end <= b.start {
break;
@@ -848,6 +872,13 @@ impl<I: VCodeInst> MachBuffer<I> {
}
self.srclocs.pop();
}
+ while let Some(last_debug_tag) = self.debug_tags.last() {
+ if last_debug_tag.offset <= b.start {
+ break;
+ }
+ self.debug_tags.pop();
+ }
+
// State:
// [PRE CODE]
// cur_off, Offset b.start, b.labels_at_this_branch:
@@ -1535,9 +1566,12 @@ impl<I: VCodeInst> MachBuffer<I> {
call_sites: self.call_sites,
exception_handlers: finalized_exception_handlers,
srclocs,
+ debug_tags: self.debug_tags,
+ debug_tag_pool: self.debug_tag_pool,
user_stack_maps: self.user_stack_maps,
unwind_info: self.unwind_info,
alignment,
+ frame_layout: self.frame_layout,
}
}
@@ -1693,6 +1727,19 @@ impl<I: VCodeInst> MachBuffer<I> {
self.user_stack_maps.push((return_addr, span, stack_map));
}
+ /// Push a debug tag associated with the current buffer offset.
+ pub fn push_debug_tags(&mut self, pos: MachDebugTagPos, tags: &[DebugTag]) {
+ trace!("debug tags at offset {}: {tags:?}", self.cur_offset());
+ let start = u32::try_from(self.debug_tag_pool.len()).unwrap();
+ self.debug_tag_pool.extend(tags.iter().cloned());
+ let end = u32::try_from(self.debug_tag_pool.len()).unwrap();
+ self.debug_tags.push(MachDebugTags {
+ offset: self.cur_offset(),
+ pos,
+ range: start..end,
+ });
+ }
+
/// Increase the alignment of the buffer to the given alignment if bigger
/// than the current alignment.
pub fn set_log2_min_function_alignment(&mut self, align_to: u8) {
@@ -1701,6 +1748,12 @@ impl<I: VCodeInst> MachBuffer<I> {
.expect("log2_min_function_alignment too large"),
);
}
+
+ /// Set the frame layout metadata.
+ pub fn set_frame_layout(&mut self, frame_layout: MachBufferFrameLayout) {
+ debug_assert!(self.frame_layout.is_none());
+ self.frame_layout = Some(frame_layout);
+ }
}
impl<I: VCodeInst> Extend<u8> for MachBuffer<I> {
@@ -1717,6 +1770,19 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
&self.srclocs[..]
}
+ /// Get all debug tags, sorted by associated offset.
+ pub fn debug_tags(&self) -> impl Iterator<Item = MachBufferDebugTagList<'_>> {
+ self.debug_tags.iter().map(|tags| {
+ let start = usize::try_from(tags.range.start).unwrap();
+ let end = usize::try_from(tags.range.end).unwrap();
+ MachBufferDebugTagList {
+ offset: tags.offset,
+ pos: tags.pos,
+ tags: &self.debug_tag_pool[start..end],
+ }
+ })
+ }
+
/// Get the total required size for the code.
pub fn total_size(&self) -> CodeOffset {
self.data.len() as CodeOffset
@@ -1792,6 +1858,11 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
}
})
}
+
+ /// Get the frame layout, if known.
+ pub fn frame_layout(&self) -> Option<&MachBufferFrameLayout> {
+ self.frame_layout.as_ref()
+ }
}
/// An item in the exception-handler list for a callsite, with label
@@ -2117,6 +2188,118 @@ impl MachBranch {
}
}
+/// Stack-frame layout information carried through to machine
+/// code. This provides sufficient information to interpret an active
+/// stack frame from a running function, if provided.
+#[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub struct MachBufferFrameLayout {
+ /// Offset from bottom of frame to FP (near top of frame). This
+ /// allows reading the frame given only FP.
+ pub frame_to_fp_offset: u32,
+ /// Offset from bottom of frame for each StackSlot,
+ pub stackslots: SecondaryMap<ir::StackSlot, MachBufferStackSlot>,
+}
+
+/// Descriptor for a single stack slot in the compiled function.
+#[derive(Clone, Debug, PartialEq, Default)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub struct MachBufferStackSlot {
+ /// Offset from the bottom of the stack frame.
+ pub offset: u32,
+
+ /// User-provided key to describe this stack slot.
+ pub key: Option<ir::StackSlotKey>,
+}
+
+/// Debug tags: a sequence of references to a stack slot, or a
+/// user-defined value, at a particular PC.
+#[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub(crate) struct MachDebugTags {
+ /// Offset at which this tag applies.
+ pub offset: CodeOffset,
+
+ /// Position on the attached instruction. This indicates whether
+ /// the tags attach to the prior instruction (i.e., as a return
+ /// point from a call) or the current instruction (i.e., as a PC
+ /// seen during a trap).
+ pub pos: MachDebugTagPos,
+
+ /// The range in the tag pool.
+ pub range: Range<u32>,
+}
+
+/// Debug tag position on an instruction.
+///
+/// We need to distinguish position on an instruction, and not just
+/// use offsets, because of the following case:
+///
+/// ```plain
+/// <tag1, tag2> call ...
+/// <tag3, tag4> trapping_store ...
+/// ```
+///
+/// If the stack is walked and interpreted with debug tags while
+/// within the call, the PC seen will be the return point, i.e. the
+/// address after the call. If the stack is walked and interpreted
+/// with debug tags upon a trap of the following instruction, it will
+/// be the PC of that instruction -- which is the same PC! Thus to
+/// disambiguate which tags we want, we attach a "pre/post" flag to
+/// every group of tags at an offset; and when we look up tags, we
+/// look them up for an offset and "position" at that offset.
+///
+/// Thus there are logically two positions at every offset -- so the
+/// above will be emitted as
+///
+/// ```plain
+/// 0: call ...
+/// 4, post: <tag1, tag2>
+/// 4, pre: <tag3, tag4>
+/// 4: trapping_store ...
+/// ```
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub enum MachDebugTagPos {
+ /// Tags attached after the instruction that ends at this offset.
+ ///
+ /// This is used to attach tags to a call, because the PC we see
+ /// when walking the stack is the *return point*.
+ Post,
+ /// Tags attached before the instruction that starts at this offset.
+ ///
+ /// This is used to attach tags to every other kind of
+ /// instruction, because the PC we see when processing a trap of
+ /// that instruction is the PC of that instruction, not the
+ /// following one.
+ Pre,
+}
+
+/// Iterator item for visiting debug tags.
+pub struct MachBufferDebugTagList<'a> {
+ /// Offset at which this tag applies.
+ pub offset: CodeOffset,
+
+ /// Position at this offset ("post", attaching to prior
+ /// instruction, or "pre", attaching to next instruction).
+ pub pos: MachDebugTagPos,
+
+ /// The underlying tags.
+ pub tags: &'a [DebugTag],
+}
+
/// Implementation of the `TextSectionBuilder` trait backed by `MachBuffer`.
///
/// Note that `MachBuffer` was primarily written for intra-function references
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index 90007f25f9..47eec0767e 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -873,6 +873,21 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
}
}
+ // If the CLIF instruction had debug tags, copy them to
+ // the VCode. Place on all VCode instructions lowered from
+ // this CLIF instruction.
+ let debug_tags = self.f.debug_tags.get(inst);
+ if !debug_tags.is_empty() && self.vcode.vcode.num_insts() > 0 {
+ let end = self.vcode.vcode.num_insts();
+ for i in start..end {
+ let backwards_index = BackwardsInsnIndex::new(i);
+ log::trace!(
+ "debug tags on {inst}; associating {debug_tags:?} with {backwards_index:?}"
+ );
+ self.vcode.add_debug_tags(backwards_index, debug_tags);
+ }
+ }
+
// maybe insert random instruction
if ctrl_plane.get_decision() {
if ctrl_plane.get_decision() {
diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs
index 38355b8cbd..77033e1c15 100644
--- a/cranelift/codegen/src/machinst/mod.rs
+++ b/cranelift/codegen/src/machinst/mod.rs
@@ -389,16 +389,10 @@ pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {
pub struct CompiledCodeBase<T: CompilePhase> {
/// Machine code.
pub buffer: MachBufferFinalized<T>,
- /// Size of stack frame, in bytes.
- pub frame_size: u32,
/// Disassembly, if requested.
pub vcode: Option<String>,
/// Debug info: value labels to registers/stackslots at code offsets.
pub value_labels_ranges: ValueLabelsRanges,
- /// Debug info: stackslots to stack pointer offsets.
- pub sized_stackslot_offsets: PrimaryMap<StackSlot, u32>,
- /// Debug info: stackslots to stack pointer offsets.
- pub dynamic_stackslot_offsets: PrimaryMap<DynamicStackSlot, u32>,
/// Basic-block layout info: block start offsets.
///
/// This info is generated only if the `machine_code_cfg_info`
@@ -418,11 +412,8 @@ impl CompiledCodeStencil {
pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode {
CompiledCode {
buffer: self.buffer.apply_base_srcloc(params.base_srcloc()),
- frame_size: self.frame_size,
vcode: self.vcode,
value_labels_ranges: self.value_labels_ranges,
- sized_stackslot_offsets: self.sized_stackslot_offsets,
- dynamic_stackslot_offsets: self.dynamic_stackslot_offsets,
bb_starts: self.bb_starts,
bb_edges: self.bb_edges,
}
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs
index cd6b8294ba..9bb44e43d5 100644
--- a/cranelift/codegen/src/machinst/vcode.rs
+++ b/cranelift/codegen/src/machinst/vcode.rs
@@ -34,6 +34,7 @@ use rustc_hash::FxHashMap;
use core::cmp::Ordering;
use core::fmt::{self, Write};
use core::mem::take;
+use core::ops::Range;
use cranelift_entity::{Keys, entity_impl};
use std::collections::HashMap;
use std::collections::hash_map::Entry;
@@ -103,6 +104,15 @@ pub struct VCode<I: VCodeInst> {
/// user stack map.
user_stack_maps: FxHashMap<BackwardsInsnIndex, ir::UserStackMap>,
+ /// A map from backwards instruction index to the debug tags for
+ /// that instruction. Each entry indexes a range in the
+ /// `debug_tag_pool`.
+ debug_tags: FxHashMap<BackwardsInsnIndex, Range<u32>>,
+
+ /// Pooled storage for sequences of debug tags; indexed by entries
+ /// in `debug_tags`.
+ debug_tag_pool: Vec<ir::DebugTag>,
+
/// Operands: pre-regalloc references to virtual registers with
/// constraints, in one flattened array. This allows the regalloc
/// to efficiently access all operands without requiring expensive
@@ -221,17 +231,8 @@ pub struct EmitResult {
/// epilogue(s), and makes use of the regalloc results.
pub disasm: Option<String>,
- /// Offsets of sized stackslots.
- pub sized_stackslot_offsets: PrimaryMap<StackSlot, u32>,
-
- /// Offsets of dynamic stackslots.
- pub dynamic_stackslot_offsets: PrimaryMap<DynamicStackSlot, u32>,
-
/// Value-labels information (debug metadata).
pub value_labels_ranges: ValueLabelsRanges,
-
- /// Stack frame size.
- pub frame_size: u32,
}
/// A builder for a VCode function body.
@@ -617,6 +618,14 @@ impl<I: VCodeInst> VCodeBuilder<I> {
let old_entry = self.vcode.user_stack_maps.insert(inst, stack_map);
debug_assert!(old_entry.is_none());
}
+
+ /// Add debug tags for the associated instruction.
+ pub fn add_debug_tags(&mut self, inst: BackwardsInsnIndex, entries: &[ir::DebugTag]) {
+ let start = u32::try_from(self.vcode.debug_tag_pool.len()).unwrap();
+ self.vcode.debug_tag_pool.extend(entries.iter().cloned());
+ let end = u32::try_from(self.vcode.debug_tag_pool.len()).unwrap();
+ self.vcode.debug_tags.insert(inst, start..end);
+ }
}
const NO_INST_OFFSET: CodeOffset = u32::MAX;
@@ -637,6 +646,8 @@ impl<I: VCodeInst> VCode<I> {
vreg_types: vec![],
insts: Vec::with_capacity(10 * n_blocks),
user_stack_maps: FxHashMap::default(),
+ debug_tags: FxHashMap::default(),
+ debug_tag_pool: vec![],
operands: Vec::with_capacity(30 * n_blocks),
operand_ranges: Ranges::with_capacity(10 * n_blocks),
clobbers: FxHashMap::default(),
@@ -938,8 +949,11 @@ impl<I: VCodeInst> VCode<I> {
// function.
let index = iix.to_backwards_insn_index(self.num_insts());
let user_stack_map = self.user_stack_maps.remove(&index);
- let user_stack_map_disasm =
- user_stack_map.as_ref().map(|m| format!(" ; {m:?}"));
+ let user_stack_map_disasm = if want_disasm {
+ user_stack_map.as_ref().map(|m| format!(" ; {m:?}"))
+ } else {
+ None
+ };
(user_stack_map, user_stack_map_disasm)
};
@@ -950,11 +964,50 @@ impl<I: VCodeInst> VCode<I> {
None
};
+ // Place debug tags in the emission buffer
+ // either at the offset prior to the
+ // instruction or after the instruction,
+ // depending on whether this is a call. See
+ // the documentation on [`MachDebugTagPos`]
+ // for details on why.
+ let mut debug_tag_disasm = None;
+ let mut place_debug_tags =
+ |this: &VCode<I>, pos: MachDebugTagPos, buffer: &mut MachBuffer<I>| {
+ // As above, translate the forward instruction
+ // index to a backward index for the lookup.
+ let debug_tag_range = {
+ let index = iix.to_backwards_insn_index(this.num_insts());
+ this.debug_tags.get(&index)
+ };
+ if let Some(range) = debug_tag_range {
+ let start = usize::try_from(range.start).unwrap();
+ let end = usize::try_from(range.end).unwrap();
+ let tags = &this.debug_tag_pool[start..end];
+
+ if want_disasm {
+ debug_tag_disasm =
+ Some(format!(" ; ^-- debug @ {pos:?}: {tags:?}"));
+ }
+ buffer.push_debug_tags(pos, tags);
+ }
+ };
+ let debug_tag_pos =
+ if self.insts[iix.index()].call_type() == CallType::Regular {
+ MachDebugTagPos::Post
+ } else {
+ MachDebugTagPos::Pre
+ };
+
+ if debug_tag_pos == MachDebugTagPos::Pre {
+ place_debug_tags(&self, debug_tag_pos, &mut buffer);
+ }
+
// If the instruction we are about to emit is
// a return, place an epilogue at this point
// (and don't emit the return; the actual
// epilogue will contain it).
if self.insts[iix.index()].is_term() == MachTerminator::Ret {
+ log::trace!("emitting epilogue");
for inst in self.abi.gen_epilogue() {
do_emit(&inst, &mut disasm, &mut buffer, &mut state);
}
@@ -981,6 +1034,8 @@ impl<I: VCodeInst> VCode<I> {
);
debug_assert!(allocs.next().is_none());
+ log::trace!("emitting: {:?}", self.insts[iix.index()]);
+
// Emit the instruction!
do_emit(
&self.insts[iix.index()],
@@ -988,10 +1043,19 @@ impl<I: VCodeInst> VCode<I> {
&mut buffer,
&mut state,
);
+
+ if debug_tag_pos == MachDebugTagPos::Post {
+ place_debug_tags(&self, debug_tag_pos, &mut buffer);
+ }
+
if let Some(stack_map_disasm) = stack_map_disasm {
disasm.push_str(&stack_map_disasm);
disasm.push('\n');
}
+ if let Some(debug_tag_disasm) = debug_tag_disasm {
+ disasm.push_str(&debug_tag_disasm);
+ disasm.push('\n');
+ }
}
}
@@ -1113,17 +1177,16 @@ impl<I: VCodeInst> VCode<I> {
self.monotonize_inst_offsets(&mut inst_offsets[..], func_body_len);
let value_labels_ranges =
self.compute_value_labels_ranges(regalloc, &inst_offsets[..], func_body_len);
- let frame_size = self.abi.frame_size();
+
+ // Store metadata about frame layout in the MachBuffer.
+ buffer.set_frame_layout(self.abi.frame_slot_metadata());
EmitResult {
buffer: buffer.finish(&self.constants, ctrl_plane),
bb_offsets,
bb_edges,
disasm: if want_disasm { Some(disasm) } else { None },
- sized_stackslot_offsets: self.abi.sized_stackslot_offsets().clone(),
- dynamic_stackslot_offsets: self.abi.dynamic_stackslot_offsets().clone(),
value_labels_ranges,
- frame_size,
}
}
diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs
index b91f1bcc04..fbb04cd771 100644
--- a/cranelift/codegen/src/verifier/mod.rs
+++ b/cranelift/codegen/src/verifier/mod.rs
@@ -2025,6 +2025,21 @@ impl<'a> Verifier<'a> {
Ok(())
}
+ pub fn debug_tags(&self, inst: Inst, errors: &mut VerifierErrors) -> VerifierStepResult {
+ // Tags can only be present on calls and sequence points.
+ let op = self.func.dfg.insts[inst].opcode();
+ let tags_allowed = op.is_call() || op == Opcode::SequencePoint;
+ let has_tags = self.func.debug_tags.has(inst);
+ if has_tags && !tags_allowed {
+ return errors.fatal((
+ inst,
+ "debug tags present on non-call, non-sequence point instruction".to_string(),
+ ));
+ }
+
+ Ok(())
+ }
+
pub fn run(&self, errors: &mut VerifierErrors) -> VerifierStepResult {
self.verify_global_values(errors)?;
self.verify_memory_types(errors)?;
@@ -2043,6 +2058,7 @@ impl<'a> Verifier<'a> {
self.typecheck(inst, errors)?;
self.immediate_constraints(inst, errors)?;
self.iconst_bounds(inst, errors)?;
+ self.debug_tags(inst, errors)?;
}
self.encodable_as_bb(block, errors)?;
diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs
index d2fbce11bb..c0025aa6fc 100644
--- a/cranelift/codegen/src/write.rs
+++ b/cranelift/codegen/src/write.rs
@@ -259,8 +259,12 @@ fn decorate_block<FW: FuncWriter>(
aliases: &SecondaryMap<Value, Vec<Value>>,
block: Block,
) -> fmt::Result {
- // Indent all instructions if any srclocs are present.
- let indent = if func.rel_srclocs().is_empty() { 4 } else { 36 };
+ // Indent all instructions if any srclocs or debug tags are present.
+ let indent = if func.rel_srclocs().is_empty() && func.debug_tags.is_empty() {
+ 4
+ } else {
+ 36
+ };
func_w.write_block_header(w, func, block, indent)?;
for a in func.dfg.block_params(block).iter().cloned() {
@@ -336,7 +340,7 @@ fn write_instruction(
func: &Function,
aliases: &SecondaryMap<Value, Vec<Value>>,
inst: Inst,
- indent: usize,
+ mut indent: usize,
) -> fmt::Result {
// Prefix containing source location, encoding, and value locations.
let mut s = String::with_capacity(16);
@@ -347,6 +351,9 @@ fn write_instruction(
write!(s, "{srcloc} ")?;
}
+ // Write out any debug tags.
+ write_debug_tags(w, &func, inst, &mut indent)?;
+
// Write out prefix and indent the instruction.
write!(w, "{s:indent$}")?;
@@ -572,6 +579,26 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
Ok(())
}
+fn write_debug_tags(
+ w: &mut dyn Write,
+ func: &Function,
+ inst: Inst,
+ indent: &mut usize,
+) -> fmt::Result {
+ let tags = func.debug_tags.get(inst);
+ if !tags.is_empty() {
+ let tags = tags
+ .iter()
+ .map(|tag| format!("{tag}"))
+ .collect::<Vec<_>>()
+ .join(", ");
+ let s = format!("<{tags}> ");
+ write!(w, "{s}")?;
+ *indent = indent.saturating_sub(s.len());
+ }
+ Ok(())
+}
+
fn write_user_stack_map_entries(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt::Result {
let entries = match dfg.user_stack_map_entries(inst) {
None => return Ok(()),
diff --git a/cranelift/filetests/filetests/inline/debug.clif b/cranelift/filetests/filetests/inline/debug.clif
new file mode 100644
index 0000000000..a6820e3a33
--- /dev/null
+++ b/cranelift/filetests/filetests/inline/debug.clif
@@ -0,0 +1,42 @@
+test inline precise-output
+target x86_64
+
+function %f0(i32, i32) -> i32 {
+ ss0 = explicit_slot 64, key = 0xfedc_ba98_7654_3210
+block0(v0: i32, v1: i32):
+<ss0, 1, 0> sequence_point
+ v2 = iadd v0, v1
+ return v2
+}
+
+; (no functions inlined into %f0)
+
+function %f1() -> i32 {
+ fn0 = %f0(i32, i32) -> i32
+ ss0 = explicit_slot 64, key = 0x1234_5678_9abc_def0
+block0():
+ v0 = iconst.i32 10
+<ss0, 2, 2> v1 = call fn0(v0, v0)
+ return v1
+}
+
+; function %f1() -> i32 fast {
+; ss0 = explicit_slot 64, key = 1311768467463790320
+; ss1 = explicit_slot 64, key = 18364758544493064720
+; sig0 = (i32, i32) -> i32 fast
+; fn0 = %f0 sig0
+;
+; block0:
+; v0 = iconst.i32 10
+; jump block1
+;
+; block1:
+; <ss0, 2, 2, ss1, 1, 0> sequence_point
+; v3 = iadd.i32 v0, v0 ; v0 = 10, v0 = 10
+; jump block2(v3)
+;
+; block2(v2: i32):
+; v1 -> v2
+; return v1
+; }
+
diff --git a/cranelift/filetests/filetests/isa/x64/debug.clif b/cranelift/filetests/filetests/isa/x64/debug.clif
new file mode 100644
index 0000000000..16fc15245e
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/debug.clif
@@ -0,0 +1,108 @@
+test compile precise-output
+target x86_64
+
+function %f0(i32, i32) -> i32 {
+ ss0 = explicit_slot 64, key = 1234
+ ;; not colocated -- shows interesting case where tags attach to
+ ;; multiple VCode insts.
+ fn0 = %f1(i32, i32)
+block0(v0: i32, v1: i32):
+<ss0, 0, 0> call fn0(v0, v1)
+<ss0, 1, 0> sequence_point
+ v2 = iadd v0, v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %r12, 0x40(%rsp)
+; movq %r14, 0x48(%rsp)
+; block0:
+; load_ext_name %f1+0, %rdx
+; ; ^-- debug @ Pre: [StackSlot(ss0), User(0), User(0)]
+; movq %rsi, %r12
+; movq %rdi, %r14
+; call *%rdx
+; ; ^-- debug @ Post: [StackSlot(ss0), User(0), User(0)]
+; sequence_point
+; ; ^-- debug @ Pre: [StackSlot(ss0), User(1), User(0)]
+; leal (%r14, %r12), %eax
+; movq 0x40(%rsp), %r12
+; movq 0x48(%rsp), %r14
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %r12, 0x40(%rsp)
+; movq %r14, 0x48(%rsp)
+; block1: ; offset 0x12
+; movabsq $0, %rdx ; reloc_external Abs8 %f1 0
+; movq %rsi, %r12
+; movq %rdi, %r14
+; callq *%rdx
+; leal (%r14, %r12), %eax
+; movq 0x40(%rsp), %r12
+; movq 0x48(%rsp), %r14
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %f1(i32, i32) -> i32 {
+ ss0 = explicit_slot 64, key = 1234
+ fn0 = colocated %f1(i32, i32)
+block0(v0: i32, v1: i32):
+<ss0, 0, 0> call fn0(v0, v1)
+<ss0, 1, 0> sequence_point
+ v2 = iadd v0, v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %rbx, 0x40(%rsp)
+; movq %r15, 0x48(%rsp)
+; block0:
+; movq %rdi, %rbx
+; movq %rsi, %r15
+; call TestCase(%f1)
+; ; ^-- debug @ Post: [StackSlot(ss0), User(0), User(0)]
+; sequence_point
+; ; ^-- debug @ Pre: [StackSlot(ss0), User(1), User(0)]
+; leal (%rbx, %r15), %eax
+; movq 0x40(%rsp), %rbx
+; movq 0x48(%rsp), %r15
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x50, %rsp
+; movq %rbx, 0x40(%rsp)
+; movq %r15, 0x48(%rsp)
+; block1: ; offset 0x12
+; movq %rdi, %rbx
+; movq %rsi, %r15
+; callq 0x1d ; reloc_external CallPCRel4 %f1 -4
+; leal (%rbx, %r15), %eax
+; movq 0x40(%rsp), %rbx
+; movq 0x48(%rsp), %r15
+; addq $0x50, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/verifier/debug_tags.clif b/cranelift/filetests/filetests/verifier/debug_tags.clif
new file mode 100644
index 0000000000..bb7fa77739
--- /dev/null
+++ b/cranelift/filetests/filetests/verifier/debug_tags.clif
@@ -0,0 +1,7 @@
+test verifier
+
+function %f(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+<0, 1> v2 = iadd.i32 v0, v1 ; error: debug tags present on non-call, non-sequence point instruction
+ return v2
+}
\ No newline at end of file
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index ac4bee433d..8a7fad2323 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -1321,6 +1321,8 @@ where
Opcode::TryCallIndirect => unimplemented!("TryCallIndirect"),
Opcode::GetExceptionHandlerAddress => unimplemented!("GetExceptionHandlerAddress"),
+
+ Opcode::SequencePoint => unimplemented!("SequencePoint"),
})
}
diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rs
index 17b3f63c22..ce8feb2215 100644
--- a/cranelift/reader/src/lexer.rs
+++ b/cranelift/reader/src/lexer.rs
@@ -19,6 +19,8 @@ pub enum Token<'a> {
RBrace, // '}'
LBracket, // '['
RBracket, // ']'
+ LAngle, // '<'
+ RAngle, // '>'
Minus, // '-'
Plus, // '+'
Multiply, // '*'
@@ -482,6 +484,8 @@ impl<'a> Lexer<'a> {
Some('}') => Some(self.scan_char(Token::RBrace)),
Some('[') => Some(self.scan_char(Token::LBracket)),
Some(']') => Some(self.scan_char(Token::RBracket)),
+ Some('<') => Some(self.scan_char(Token::LAngle)),
+ Some('>') => Some(self.scan_char(Token::RAngle)),
Some(',') => Some(self.scan_char(Token::Comma)),
Some('.') => Some(self.scan_char(Token::Dot)),
Some(':') => Some(self.scan_char(Token::Colon)),
diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs
index 29faadee23..e2172ae99b 100644
--- a/cranelift/reader/src/parser.rs
+++ b/cranelift/reader/src/parser.rs
@@ -15,8 +15,8 @@ use cranelift_codegen::ir::immediates::{
};
use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs};
use cranelift_codegen::ir::pcc::{BaseExpr, Expr, Fact};
-use cranelift_codegen::ir::types::*;
-use cranelift_codegen::ir::{self, UserExternalNameRef};
+use cranelift_codegen::ir::{self, StackSlotKey, UserExternalNameRef};
+use cranelift_codegen::ir::{DebugTag, types::*};
use cranelift_codegen::ir::{
AbiParam, ArgumentExtension, ArgumentPurpose, Block, BlockArg, Constant, ConstantData,
@@ -956,6 +956,38 @@ impl<'a> Parser<'a> {
}
}
+ /// Parse an optional list of debug tags.
+ fn optional_debug_tags(&mut self) -> ParseResult<Vec<DebugTag>> {
+ if self.optional(Token::LAngle) {
+ let mut tags = vec![];
+ while !self.optional(Token::RAngle) {
+ match self.token() {
+ Some(Token::Integer(_)) => {
+ let value: u32 = self.match_uimm32("expected a u32 value")?.into();
+ tags.push(DebugTag::User(value));
+ }
+ Some(Token::StackSlot(slot)) => {
+ self.consume();
+ tags.push(DebugTag::StackSlot(StackSlot::from_u32(slot)));
+ }
+ _ => {
+ return err!(
+ self.loc,
+ "expected integer user value or stack slot in debug tags"
+ );
+ }
+ }
+ if !self.optional(Token::Comma) {
+ self.match_token(Token::RAngle, "expected `,` or `>`")?;
+ break;
+ }
+ }
+ Ok(tags)
+ } else {
+ Ok(vec![])
+ }
+ }
+
/// Parse a list of literals (i.e. integers, floats, booleans); e.g. `0 1 2 3`, usually as
/// part of something like `vconst.i32x4 [0 1 2 3]`.
fn parse_literals_to_constant_data(&mut self, ty: Type) -> ParseResult<ConstantData> {
@@ -1510,7 +1542,7 @@ impl<'a> Parser<'a> {
// | "spill_slot"
// | "incoming_arg"
// | "outgoing_arg"
- // stack-slot-flag ::= "align" "=" Bytes
+ // stack-slot-flag ::= "align" "=" Bytes | "key" "=" uimm64
fn parse_stack_slot_decl(&mut self) -> ParseResult<(StackSlot, StackSlotData)> {
let ss = self.match_ss("expected stack slot number: ss«n»")?;
self.match_token(Token::Equal, "expected '=' in stack slot declaration")?;
@@ -1527,29 +1559,42 @@ impl<'a> Parser<'a> {
return err!(self.loc, "stack slot too large");
}
- // Parse flags.
- let align = if self.token() == Some(Token::Comma) {
+ let mut align = 1;
+ let mut key = None;
+
+ while self.token() == Some(Token::Comma) {
self.consume();
- self.match_token(
- Token::Identifier("align"),
- "expected a valid stack-slot flag (currently only `align`)",
- )?;
- self.match_token(Token::Equal, "expected `=` after flag")?;
- let align: i64 = self
- .match_imm64("expected alignment-size after `align` flag")?
- .into();
- u32::try_from(align)
- .map_err(|_| self.error("alignment must be a 32-bit unsigned integer"))?
- } else {
- 1
- };
+ match self.token() {
+ Some(Token::Identifier("align")) => {
+ self.consume();
+ self.match_token(Token::Equal, "expected `=` after flag")?;
+ let align64: i64 = self
+ .match_imm64("expected alignment-size after `align` flag")?
+ .into();
+ align = u32::try_from(align64)
+ .map_err(|_| self.error("alignment must be a 32-bit unsigned integer"))?;
+ }
+ Some(Token::Identifier("key")) => {
+ self.consume();
+ self.match_token(Token::Equal, "expected `=` after flag")?;
+ let value = self.match_uimm64("expected `u64` value for `key` flag")?;
+ key = Some(StackSlotKey::new(value.into()));
+ }
+ _ => {
+ return Err(self.error("invalid flag for stack slot"));
+ }
+ }
+ }
if !align.is_power_of_two() {
return err!(self.loc, "stack slot alignment is not a power of two");
}
let align_shift = u8::try_from(align.ilog2()).unwrap(); // Always succeeds: range 0..=31.
- let data = StackSlotData::new(kind, bytes as u32, align_shift);
+ let data = match key {
+ Some(key) => StackSlotData::new_with_key(kind, bytes as u32, align_shift, key),
+ None => StackSlotData::new(kind, bytes as u32, align_shift),
+ };
// Collect any trailing comments.
self.token();
@@ -2103,11 +2148,14 @@ impl<'a> Parser<'a> {
Some(Token::Value(_))
| Some(Token::Identifier(_))
| Some(Token::LBracket)
- | Some(Token::SourceLoc(_)) => true,
+ | Some(Token::SourceLoc(_))
+ | Some(Token::LAngle) => true,
_ => false,
} {
let srcloc = self.optional_srcloc()?;
+ let debug_tags = self.optional_debug_tags()?;
+
// We need to parse instruction results here because they are shared
// between the parsing of value aliases and the parsing of instructions.
//
@@ -2127,10 +2175,10 @@ impl<'a> Parser<'a> {
}
Some(Token::Equal) => {
self.consume();
- self.parse_instruction(&results, srcloc, ctx, block)?;
+ self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?;
}
_ if !results.is_empty() => return err!(self.loc, "expected -> or ="),
- _ => self.parse_instruction(&results, srcloc, ctx, block)?,
+ _ => self.parse_instruction(&results, srcloc, debug_tags, ctx, block)?,
}
}
@@ -2521,6 +2569,7 @@ impl<'a> Parser<'a> {
&mut self,
results: &[Value],
srcloc: ir::SourceLoc,
+ debug_tags: Vec<DebugTag>,
ctx: &mut Context,
block: Block,
) -> ParseResult<()> {
@@ -2560,16 +2609,22 @@ impl<'a> Parser<'a> {
// instruction ::= [inst-results "="] Opcode(opc) ["." Type] * ...
let inst_data = self.parse_inst_operands(ctx, opcode, explicit_ctrl_type)?;
- // We're done parsing the instruction data itself.
- //
- // We still need to check that the number of result values in the source
- // matches the opcode or function call signature. We also need to create
- // values with the right type for all the instruction results and parse
- // and attach stack map entries, if present.
let ctrl_typevar = self.infer_typevar(ctx, opcode, explicit_ctrl_type, &inst_data)?;
let inst = ctx.function.dfg.make_inst(inst_data);
- if opcode.is_call() && !opcode.is_return() && self.optional(Token::Comma) {
- self.match_identifier("stack_map", "expected `stack_map = [...]`")?;
+
+ // Attach stack map, if present.
+ if self.optional(Token::Comma) {
+ self.match_token(
+ Token::Identifier("stack_map"),
+ "expected `stack_map = [...]`",
+ )?;
+ if !opcode.is_call() || opcode.is_return() {
+ return err!(
+ self.loc,
+ "stack map can only be attached to a (non-tail) call"
+ );
+ }
+
self.match_token(Token::Equal, "expected `= [...]`")?;
self.match_token(Token::LBracket, "expected `[...]`")?;
while !self.optional(Token::RBracket) {
@@ -2594,6 +2649,13 @@ impl<'a> Parser<'a> {
}
}
}
+
+ // We're done parsing the instruction data itself.
+ //
+ // We still need to check that the number of result values in
+ // the source matches the opcode or function call
+ // signature. We also need to create values with the right
+ // type for all the instruction results.
let num_results =
ctx.function
.dfg
@@ -2606,6 +2668,9 @@ impl<'a> Parser<'a> {
if !srcloc.is_default() {
ctx.function.set_srcloc(inst, srcloc);
}
+ if !debug_tags.is_empty() {
+ ctx.function.debug_tags.set(inst, debug_tags);
+ }
if results.len() != num_results {
return err!(