Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: add get_exception_handler_address. (#11629)

Chris Fallin4c01ee2f2c1f
diff --git a/cranelift/codegen/meta/src/cdsl/formats.rs b/cranelift/codegen/meta/src/cdsl/formats.rs
index 10d804a98a..f9cec838cf 100644
--- a/cranelift/codegen/meta/src/cdsl/formats.rs
+++ b/cranelift/codegen/meta/src/cdsl/formats.rs
@@ -40,6 +40,8 @@ pub(crate) struct InstructionFormat {
pub num_block_operands: usize,
+ pub num_raw_block_operands: usize,
+
/// Index of the value input operand that is used to infer the controlling type variable. By
/// default, this is `0`, the first `value` operand. The index is relative to the values only,
/// ignoring immediate operands.
@@ -52,6 +54,7 @@ pub(crate) struct FormatStructure {
pub num_value_operands: usize,
pub has_value_list: bool,
pub num_block_operands: usize,
+ pub num_raw_block_operands: usize,
/// Tuples of (Rust field name / Rust type) for each immediate field.
pub imm_field_names: Vec<(&'static str, &'static str)>,
}
@@ -65,8 +68,12 @@ impl fmt::Display for InstructionFormat {
.collect::<Vec<_>>()
.join(", ");
fmt.write_fmt(format_args!(
- "{}(imms=({}), vals={}, blocks={})",
- self.name, imm_args, self.num_value_operands, self.num_block_operands,
+ "{}(imms=({}), vals={}, blocks={}, raw_blocks={})",
+ self.name,
+ imm_args,
+ self.num_value_operands,
+ self.num_block_operands,
+ self.num_raw_block_operands,
))?;
Ok(())
}
@@ -79,6 +86,7 @@ impl InstructionFormat {
num_value_operands: self.num_value_operands,
has_value_list: self.has_value_list,
num_block_operands: self.num_block_operands,
+ num_raw_block_operands: self.num_raw_block_operands,
imm_field_names: self
.imm_fields
.iter()
@@ -97,6 +105,7 @@ impl InstructionFormatBuilder {
num_value_operands: 0,
has_value_list: false,
num_block_operands: 0,
+ num_raw_block_operands: 0,
imm_fields: Vec::new(),
typevar_operand: None,
})
@@ -117,6 +126,11 @@ impl InstructionFormatBuilder {
self
}
+ pub fn raw_block(mut self) -> Self {
+ self.0.num_raw_block_operands += 1;
+ self
+ }
+
pub fn imm(mut self, operand_kind: &OperandKind) -> Self {
let field = FormatField {
kind: operand_kind.clone(),
diff --git a/cranelift/codegen/meta/src/cdsl/instructions.rs b/cranelift/codegen/meta/src/cdsl/instructions.rs
index b944255149..49293cfc13 100644
--- a/cranelift/codegen/meta/src/cdsl/instructions.rs
+++ b/cranelift/codegen/meta/src/cdsl/instructions.rs
@@ -288,6 +288,7 @@ fn verify_format(inst_name: &str, operands_in: &[Operand], format: &InstructionF
// - whether it has a value list or not.
let mut num_values = 0;
let mut num_blocks = 0;
+ let mut num_raw_blocks = 0;
let mut num_immediates = 0;
for operand in operands_in.iter() {
@@ -304,6 +305,8 @@ fn verify_format(inst_name: &str, operands_in: &[Operand], format: &InstructionF
}
if operand.kind.is_block() {
num_blocks += 1;
+ } else if operand.kind.is_raw_block() {
+ num_raw_blocks += 1;
} else if operand.is_immediate_or_entityref() {
if let Some(format_field) = format.imm_fields.get(num_immediates) {
assert_eq!(
@@ -335,6 +338,13 @@ fn verify_format(inst_name: &str, operands_in: &[Operand], format: &InstructionF
inst_name, format.name,
);
+ assert_eq!(
+ num_raw_blocks, format.num_raw_block_operands,
+ "inst {} doesn't have as many raw-block input operands as its format {} declares; you may need \
+ to use a different format.",
+ inst_name, format.name,
+ );
+
assert_eq!(
num_immediates,
format.imm_fields.len(),
diff --git a/cranelift/codegen/meta/src/cdsl/operands.rs b/cranelift/codegen/meta/src/cdsl/operands.rs
index f2ec8f9d47..6cd149a8da 100644
--- a/cranelift/codegen/meta/src/cdsl/operands.rs
+++ b/cranelift/codegen/meta/src/cdsl/operands.rs
@@ -150,6 +150,10 @@ impl OperandKind {
pub(crate) fn is_block(&self) -> bool {
self.rust_type == "ir::BlockCall"
}
+
+ pub(crate) fn is_raw_block(&self) -> bool {
+ self.rust_type == "ir::Block"
+ }
}
impl From<&TypeVar> for OperandKind {
diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index 106333d561..ab0db7c757 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -87,6 +87,12 @@ fn gen_instruction_data(formats: &[Rc<InstructionFormat>], fmt: &mut Formatter)
n => panic!("Too many block operands in instruction: {n}"),
}
+ match format.num_raw_block_operands {
+ 0 => (),
+ 1 => fmt.line("block: ir::Block,"),
+ n => panic!("Too many block operands in instruction: {n}"),
+ }
+
for field in &format.imm_fields {
fmtln!(fmt, "{}: {},", field.member, field.kind.rust_type);
}
@@ -249,6 +255,15 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
}
};
+ let raw_blocks_eq = match format.num_raw_block_operands {
+ 0 => None,
+ 1 => {
+ members.push("block");
+ Some("block1 == block2")
+ }
+ _ => unreachable!("Not a valid format"),
+ };
+
for field in &format.imm_fields {
members.push(field.member);
}
@@ -266,6 +281,9 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
if let Some(blocks_eq) = blocks_eq {
fmtln!(fmt, "&& {}", blocks_eq);
}
+ if let Some(raw_blocks_eq) = raw_blocks_eq {
+ fmtln!(fmt, "&& {}", raw_blocks_eq);
+ }
});
}
fmt.line("_ => unreachable!()");
@@ -314,6 +332,15 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
}
};
+ let raw_block = match format.num_raw_block_operands {
+ 0 => None,
+ 1 => {
+ members.push("block");
+ Some("block")
+ }
+ _ => panic!("Too many raw block operands"),
+ };
+
for field in &format.imm_fields {
members.push(field.member);
}
@@ -341,6 +368,10 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
});
});
}
+
+ if let Some(raw_block) = raw_block {
+ fmtln!(fmt, "::core::hash::Hash::hash(&{raw_block}, state);");
+ }
});
}
});
@@ -378,6 +409,14 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
}
};
+ match format.num_raw_block_operands {
+ 0 => {}
+ 1 => {
+ members.push("block");
+ }
+ _ => panic!("Too many raw-block operands to format"),
+ }
+
for field in &format.imm_fields {
members.push(field.member);
}
@@ -406,6 +445,14 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
_ => panic!("Too many block targets in instruction"),
}
+ match format.num_raw_block_operands {
+ 0 => {}
+ 1 => {
+ fmtln!(fmt, "block,");
+ }
+ _ => panic!("Too many raw-block operands in instruction"),
+ }
+
for field in &format.imm_fields {
fmtln!(fmt, "{},", field.member);
}
@@ -442,6 +489,14 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
}
};
+ match format.num_raw_block_operands {
+ 0 => {}
+ 1 => {
+ members.push("block");
+ }
+ _ => panic!("Too many raw-block operands"),
+ }
+
for field in &format.imm_fields {
members.push(field.member);
}
@@ -474,6 +529,14 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
_ => panic!("Too many block targets in instruction"),
}
+ match format.num_raw_block_operands {
+ 0 => {}
+ 1 => {
+ fmtln!(fmt, "block: mapper.map_block(block),");
+ }
+ _ => panic!("Too many raw block arguments in instruction"),
+ }
+
for field in &format.imm_fields {
let member = field.member;
match &field.kind.fields {
@@ -958,6 +1021,13 @@ fn gen_member_inits(format: &InstructionFormat, fmt: &mut Formatter) {
fmtln!(fmt, "blocks: [{}],", blocks.join(", "));
}
}
+
+ // Raw block operands.
+ match format.num_raw_block_operands {
+ 0 => (),
+ 1 => fmt.line("block: block0,"),
+ _ => panic!("Too many raw block arguments"),
+ }
}
/// Emit a method for creating and inserting an instruction format.
@@ -972,6 +1042,9 @@ fn gen_format_constructor(format: &InstructionFormat, fmt: &mut Formatter) {
"ctrl_typevar: Type".into(),
];
+ // Raw block operands.
+ args.extend((0..format.num_raw_block_operands).map(|i| format!("block{i}: ir::Block")));
+
// Normal operand arguments. Start with the immediate operands.
for f in &format.imm_fields {
args.push(format!("{}: {}", f.member, f.kind.rust_type));
@@ -1074,6 +1147,9 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo
args_doc.push(format!("- {}_args: {}", op.name, "Block arguments"));
block_args.push(op);
+ } else if op.kind.is_raw_block() {
+ args.push("block: ir::Block".into());
+ args_doc.push("- block: raw basic block".into());
} else {
let t = if op.is_immediate() {
let t = format!("T{}", tmpl_types.len() + 1);
diff --git a/cranelift/codegen/meta/src/gen_isle.rs b/cranelift/codegen/meta/src/gen_isle.rs
index 338f570976..79755ca363 100644
--- a/cranelift/codegen/meta/src/gen_isle.rs
+++ b/cranelift/codegen/meta/src/gen_isle.rs
@@ -142,6 +142,10 @@ fn gen_common_isle(
fmt.empty_line();
}
+ // Raw block entities.
+ fmtln!(fmt, "(type Block extern (enum))");
+ fmt.empty_line();
+
// Generate the extern type declaration for `Opcode`.
fmt.line(";;;; `Opcode` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;");
fmt.empty_line();
@@ -184,6 +188,12 @@ fn gen_common_isle(
n => write!(&mut s, " (blocks BlockArray{n})").unwrap(),
}
+ match format.num_raw_block_operands {
+ 0 => (),
+ 1 => write!(&mut s, "(block Block)").unwrap(),
+ _ => panic!("Too many raw block arguments"),
+ }
+
for field in &format.imm_fields {
write!(
&mut s,
@@ -322,7 +332,9 @@ fn gen_common_isle(
let imm_operands: Vec<_> = inst
.operands_in
.iter()
- .filter(|o| !o.is_value() && !o.is_varargs() && !o.kind.is_block())
+ .filter(|o| {
+ !o.is_value() && !o.is_varargs() && !o.kind.is_block() && !o.kind.is_raw_block()
+ })
.collect();
assert_eq!(imm_operands.len(), inst.format.imm_fields.len(),);
for op in imm_operands {
@@ -353,6 +365,15 @@ fn gen_common_isle(
}
}
+ // Raw blocks.
+ match inst.format.num_raw_block_operands {
+ 0 => {}
+ 1 => {
+ write!(&mut s, " block").unwrap();
+ }
+ _ => panic!("Too many raw block arguments"),
+ }
+
s.push_str("))");
fmt.line(&s);
});
@@ -441,12 +462,18 @@ fn gen_common_isle(
}
}
+ match inst.format.num_raw_block_operands {
+ 0 => {}
+ 1 => {
+ write!(&mut s, " block").unwrap();
+ }
+ _ => panic!("Too many raw block arguments"),
+ }
+
// Immediates (non-value args).
- for o in inst
- .operands_in
- .iter()
- .filter(|o| !o.is_value() && !o.is_varargs() && !o.kind.is_block())
- {
+ for o in inst.operands_in.iter().filter(|o| {
+ !o.is_value() && !o.is_varargs() && !o.kind.is_block() && !o.kind.is_raw_block()
+ }) {
write!(&mut s, " {}", o.name).unwrap();
}
s.push_str("))");
diff --git a/cranelift/codegen/meta/src/shared/entities.rs b/cranelift/codegen/meta/src/shared/entities.rs
index e707d67672..26289bcfaa 100644
--- a/cranelift/codegen/meta/src/shared/entities.rs
+++ b/cranelift/codegen/meta/src/shared/entities.rs
@@ -23,6 +23,11 @@ pub(crate) struct EntityRefs {
/// This is primarily used in control flow instructions.
pub(crate) block_else: OperandKind,
+ /// A reference to a basic block in the same function, without any arguments.
+ /// This is primarily used to refer to block `try_call` terminators to get
+ /// exception metadata (e.g., resume PCs) as first-class values.
+ pub(crate) raw_block: OperandKind,
+
/// A reference to a stack slot declared in the function preamble.
pub(crate) stack_slot: OperandKind,
@@ -84,6 +89,12 @@ impl EntityRefs {
"a basic block in the same function, with its arguments provided.",
),
+ raw_block: new(
+ "raw_block",
+ "ir::Block",
+ "a basic block in the same function, with no arguments provided.",
+ ),
+
stack_slot: new("stack_slot", "ir::StackSlot", "A stack slot"),
dynamic_stack_slot: new(
diff --git a/cranelift/codegen/meta/src/shared/formats.rs b/cranelift/codegen/meta/src/shared/formats.rs
index 3c03ab0bc6..5994b0b583 100644
--- a/cranelift/codegen/meta/src/shared/formats.rs
+++ b/cranelift/codegen/meta/src/shared/formats.rs
@@ -42,6 +42,7 @@ pub(crate) struct Formats {
pub(crate) unary_ieee32: Rc<InstructionFormat>,
pub(crate) unary_ieee64: Rc<InstructionFormat>,
pub(crate) unary_imm: Rc<InstructionFormat>,
+ pub(crate) exception_handler_address: Rc<InstructionFormat>,
}
impl Formats {
@@ -219,6 +220,11 @@ impl Formats {
.value()
.imm(&imm.trapcode)
.build(),
+
+ exception_handler_address: Builder::new("ExceptionHandlerAddress")
+ .raw_block()
+ .imm(&imm.imm64)
+ .build(),
}
}
}
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs
index a8035cb67b..7480a4d44e 100644
--- a/cranelift/codegen/meta/src/shared/instructions.rs
+++ b/cranelift/codegen/meta/src/shared/instructions.rs
@@ -1403,6 +1403,38 @@ pub(crate) fn define(
.operands_out(vec![Operand::new("addr", iAddr)]),
);
+ ig.push(
+ Inst::new(
+ "get_exception_handler_address",
+ r#"
+ Get the handler PC for the given exceptional edge for an
+ exception return from the given `try_call`-terminated block.
+
+ This instruction provides the PC for the handler resume point,
+ as defined by the exception-handling aspect of the given
+ callee ABI, for a return from the given calling block. It can
+ be used when the exception unwind mechanism requires manual
+ plumbing for this information which must be set up before the call
+ itself: for example, if the resume address needs to be stored in
+ some context structure for a runtime to resume to on error.
+
+ The given caller block must end in a `try_call` and the given
+ exception-handling block must be one of its exceptional
+ successors in the associated exception-handling table. The
+ returned PC is *only* valid to resume to when the `try_call`
+ is on the stack having called the callee; in other words, when
+ a normal exception unwinder might otherwise resume to that
+ handler.
+ "#,
+ &formats.exception_handler_address,
+ )
+ .operands_in(vec![
+ Operand::new("block", &entities.raw_block),
+ Operand::new("index", &imm.imm64),
+ ])
+ .operands_out(vec![Operand::new("addr", iAddr)]),
+ );
+
ig.push(
Inst::new(
"iconst",
diff --git a/cranelift/codegen/src/inline.rs b/cranelift/codegen/src/inline.rs
index b05475c520..9a425086fd 100644
--- a/cranelift/codegen/src/inline.rs
+++ b/cranelift/codegen/src/inline.rs
@@ -946,6 +946,10 @@ impl<'a> ir::instructions::InstructionMapper for InliningInstRemapper<'a> {
ir::BlockCall::new(inlined_block, args, &mut self.func.dfg.value_lists)
}
+ fn map_block(&mut self, block: ir::Block) -> ir::Block {
+ self.entity_map.inlined_block(block)
+ }
+
fn map_func_ref(&mut self, func_ref: ir::FuncRef) -> ir::FuncRef {
self.entity_map.inlined_func_ref(func_ref)
}
diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs
index c10cc33750..2b94d9da1e 100644
--- a/cranelift/codegen/src/ir/instructions.rs
+++ b/cranelift/codegen/src/ir/instructions.rs
@@ -1061,6 +1061,9 @@ pub trait InstructionMapper {
/// Map a function over a `BlockCall`.
fn map_block_call(&mut self, block_call: BlockCall) -> BlockCall;
+ /// Map a function over a `Block`.
+ fn map_block(&mut self, block: Block) -> Block;
+
/// Map a function over a `FuncRef`.
fn map_func_ref(&mut self, func_ref: FuncRef) -> FuncRef;
@@ -1111,6 +1114,10 @@ where
(**self).map_block_call(block_call)
}
+ fn map_block(&mut self, block: Block) -> Block {
+ (**self).map_block(block)
+ }
+
fn map_func_ref(&mut self, func_ref: FuncRef) -> FuncRef {
(**self).map_func_ref(func_ref)
}
@@ -1329,6 +1336,10 @@ mod tests {
BlockCall::new(block, [], &mut pool)
}
+ fn map_block(&mut self, block: Block) -> Block {
+ Block::from_u32(block.as_u32() + 1)
+ }
+
fn map_func_ref(&mut self, func_ref: FuncRef) -> FuncRef {
FuncRef::from_u32(func_ref.as_u32() + 1)
}
@@ -1446,6 +1457,20 @@ mod tests {
})
);
+ // Mapping `Block`s.
+ assert_eq!(
+ map(InstructionData::ExceptionHandlerAddress {
+ opcode: Opcode::GetExceptionHandlerAddress,
+ block: Block::from_u32(1),
+ imm: 0.into(),
+ }),
+ InstructionData::ExceptionHandlerAddress {
+ opcode: Opcode::GetExceptionHandlerAddress,
+ block: Block::from_u32(2),
+ imm: 0.into(),
+ },
+ );
+
// Mapping `SigRef`s.
assert_eq!(
map(InstructionData::CallIndirect {
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 0baa5e3235..14efbfef42 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -993,6 +993,10 @@
(DummyUse
(reg Reg))
+ ;; A pseudoinstruction that loads the address of a label.
+ (LabelAddress (dst WritableReg)
+ (label MachLabel))
+
;; Emits an inline stack probe loop.
;;
;; Note that this is emitted post-regalloc so `start` and `end` can be
@@ -5182,3 +5186,10 @@
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.FpuLoad128 dst amode flags))))
dst))
+
+;; Helper for creating an `LabelAddress` instruction.
+(decl a64_label_address (MachLabel) Reg)
+(rule (a64_label_address label)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LabelAddress dst label))))
+ dst))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 13ffccb38f..8310c41609 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -3535,6 +3535,17 @@ impl MachInstEmit for Inst {
&Inst::DummyUse { .. } => {}
+ &Inst::LabelAddress { dst, label } => {
+ // We emit an ADR only, which is +/- 2MiB range. This
+ // should be sufficient for the typical use-case of
+ // this instruction, which is insmall trampolines to
+ // get exception-handler addresses.
+ let inst = Inst::Adr { rd: dst, off: 0 };
+ let offset = sink.cur_offset();
+ inst.emit(sink, emit_info, state);
+ sink.use_label_at_offset(offset, label, LabelUse::Adr21);
+ }
+
&Inst::StackProbeLoop { start, end, step } => {
assert!(emit_info.0.enable_probestack());
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 9655e3e59a..fdce52eb46 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -916,6 +916,9 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
Inst::DummyUse { reg } => {
collector.reg_use(reg);
}
+ Inst::LabelAddress { dst, .. } => {
+ collector.reg_def(dst);
+ }
Inst::StackProbeLoop { start, end, .. } => {
collector.reg_early_def(start);
collector.reg_use(end);
@@ -2887,6 +2890,10 @@ impl Inst {
let reg = pretty_print_reg(reg);
format!("dummy_use {reg}")
}
+ &Inst::LabelAddress { dst, label } => {
+ let dst = pretty_print_reg(dst.to_reg());
+ format!("label_address {dst}, {label:?}")
+ }
&Inst::StackProbeLoop { start, end, step } => {
let start = pretty_print_reg(start.to_reg());
let end = pretty_print_reg(end);
@@ -2983,7 +2990,9 @@ impl MachInstLabelUse for LabelUse {
LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
- LabelUse::Adr21 => (pc_rel_shifted & 0x7ffff) << 5 | (pc_rel_shifted & 0x180000) << 10,
+ // Note: the *low* two bits of offset are put in the
+ // *high* bits (30, 29).
+ LabelUse::Adr21 => (pc_rel_shifted & 0x1ffffc) << 3 | (pc_rel_shifted & 3) << 29,
LabelUse::PCRel32 => pc_rel_shifted,
};
let is_add = match self {
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 8c43493354..4efaac2dbc 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -3238,3 +3238,9 @@
(emit_island (targets_jt_space targets))))
(ridx Reg (put_in_reg_zext32 idx)))
(br_table_impl jt_size ridx default targets)))
+
+;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
+ (let ((succ_label MachLabel (block_exn_successor_label block idx)))
+ (a64_label_address succ_label)))
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle
index 55edfef133..4f9faf02ee 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle
+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle
@@ -91,6 +91,10 @@
;; Island generation prior to variable-length instructions.
(EmitIsland (space_needed u32))
+
+ ;; A pseudoinstruction that loads the address of a label.
+ (LabelAddress (dst WritableXReg)
+ (label MachLabel))
)
)
@@ -781,3 +785,10 @@
(rule (sext64 val @ (value_type $I16)) (pulley_sext16 val))
(rule (sext64 val @ (value_type $I32)) (pulley_sext32 val))
(rule (sext64 val @ (value_type $I64)) val)
+
+;; Helper for creating an `LabelAddress` instruction.
+(decl pulley_label_address (MachLabel) Reg)
+(rule (pulley_label_address label)
+ (let ((dst WritableReg (temp_writable_xreg))
+ (_ Unit (emit (MInst.LabelAddress dst label))))
+ dst))
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
index dd8ad82835..08d8b5eaf7 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
@@ -590,6 +590,12 @@ fn pulley_emit<P>(
sink.bind_label(label, &mut state.ctrl_plane);
}
}
+
+ Inst::LabelAddress { dst, label } => {
+ patch_pc_rel_offset(sink, |sink| enc::xpcadd(sink, dst, 0));
+ let end = sink.cur_offset();
+ sink.use_label_at_offset(end - 4, *label, LabelUse::PcRel);
+ }
}
}
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
index 40bca35738..bc2490f7bc 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
@@ -331,6 +331,10 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
Inst::Raw { raw } => generated::get_operands(raw, collector),
Inst::EmitIsland { .. } => {}
+
+ Inst::LabelAddress { dst, label: _ } => {
+ collector.reg_def(dst);
+ }
}
}
@@ -825,6 +829,11 @@ impl Inst {
Inst::Raw { raw } => generated::print(raw),
Inst::EmitIsland { space_needed } => format!("emit_island {space_needed}"),
+
+ Inst::LabelAddress { dst, label } => {
+ let dst = format_reg(dst.to_reg().to_reg());
+ format!("label_address {dst}, {label:?}")
+ }
}
}
}
diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle
index 873cab0403..12770d844e 100644
--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle
+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle
@@ -1829,3 +1829,10 @@
(rule (lower (func_addr (func_ref_data _ extname dist)))
(load_ext_name extname 0 dist))
+
+
+;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
+ (let ((succ_label MachLabel (block_exn_successor_label block idx)))
+ (pulley_label_address succ_label)))
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 6e020d6a24..f118fcf710 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -254,6 +254,10 @@
(DummyUse
(reg Reg))
+ ;; A pseudoinstruction that loads the address of a label.
+ (LabelAddress (dst WritableReg)
+ (label MachLabel))
+
;; popcnt if target doesn't support extension B
;; use iteration to implement.
(Popcnt
@@ -3250,3 +3254,11 @@
;; a >= b
(rule 1 (fcmp_to_float_compare (FloatCC.GreaterThanOrEqual) ty a b)
(FloatCompare.One (rv_fge ty a b)))
+
+
+;; Helper for creating an `LabelAddress` instruction.
+(decl rv64_label_address (MachLabel) Reg)
+(rule (rv64_label_address label)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LabelAddress dst label))))
+ dst))
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index cfc062dc10..e7a64b605f 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -194,6 +194,7 @@ impl Inst {
| Inst::TrapIf { .. }
| Inst::Unwind { .. }
| Inst::DummyUse { .. }
+ | Inst::LabelAddress { .. }
| Inst::Popcnt { .. }
| Inst::Cltz { .. }
| Inst::Brev8 { .. }
@@ -2106,6 +2107,26 @@ impl Inst {
.emit_uncompressed(sink, emit_info, state, start_off);
}
+ &Inst::LabelAddress { dst, label } => {
+ let offset = sink.cur_offset();
+ Inst::Auipc {
+ rd: dst,
+ imm: Imm20::from_i32(0),
+ }
+ .emit_uncompressed(sink, emit_info, state, start_off);
+ sink.use_label_at_offset(offset, label, LabelUse::PCRelHi20);
+
+ let offset = sink.cur_offset();
+ Inst::AluRRImm12 {
+ alu_op: AluOPRRI::Addi,
+ rd: dst,
+ rs: dst.to_reg(),
+ imm12: Imm12::ZERO,
+ }
+ .emit_uncompressed(sink, emit_info, state, start_off);
+ sink.use_label_at_offset(offset, label, LabelUse::PCRelLo12I);
+ }
+
&Inst::ElfTlsGetAddr { rd, ref name } => {
// RISC-V's TLS GD model is slightly different from other arches.
//
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
index 395e04270f..5f4c037b29 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -696,6 +696,9 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
vec_mask_operands(mask, collector);
}
Inst::EmitIsland { .. } => {}
+ Inst::LabelAddress { dst, .. } => {
+ collector.reg_def(dst);
+ }
}
}
@@ -1672,6 +1675,11 @@ impl Inst {
Inst::EmitIsland { needed_space } => {
format!("emit_island {needed_space}")
}
+
+ Inst::LabelAddress { dst, label } => {
+ let dst = format_reg(dst.to_reg());
+ format!("label_address {dst}, {label:?}")
+ }
}
}
}
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index a197c4990a..d2329133b7 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -3130,4 +3130,11 @@
(y_pos VReg (rv_vmax_vx y (zero_reg) (unmasked) in_ty))
(x_clip VReg (rv_vnclipu_wi x_pos zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))
(y_clip VReg (rv_vnclipu_wi y_pos zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty)))))
- (rv_vslideup_vvi x_clip y_clip lane_diff (unmasked) out_ty)))
\ No newline at end of file
+ (rv_vslideup_vvi x_clip y_clip lane_diff (unmasked) out_ty)))
+
+
+;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
+ (let ((succ_label MachLabel (block_exn_successor_label block idx)))
+ (rv64_label_address succ_label)))
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 57f70f65d1..91d0227b3c 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -1015,6 +1015,10 @@
(got Reg)
(got_offset Reg)
(symbol BoxSymbolReloc))
+
+ ;; A pseudoinstruction that loads the address of a label.
+ (LabelAddress (dst WritableReg)
+ (label MachLabel))
))
;; Primitive types used in instruction formats.
@@ -4928,6 +4932,14 @@
(decl vec_fcmphes (Type Reg Reg) ProducesFlags)
(rule (vec_fcmphes (ty_vec128 ty) x y) (vec_float_cmps ty (vecop_float_cmphe ty) x y))
+;; Other helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Helper for creating an `LabelAddress` instruction.
+(decl s390x_label_address (MachLabel) Reg)
+(rule (s390x_label_address label)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LabelAddress dst label))))
+ dst))
;; Implicit conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index a82d46f1e7..98718d1ee1 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -3536,6 +3536,14 @@ impl Inst {
}
&Inst::DummyUse { .. } => {}
+
+ &Inst::LabelAddress { dst, label } => {
+ let inst = Inst::LoadAddr {
+ rd: dst,
+ mem: MemArg::Label { target: label },
+ };
+ inst.emit(sink, emit_info, state);
+ }
}
state.clear_post_insn();
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 2e1a2f3627..256de41504 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -282,6 +282,8 @@ impl Inst {
| Inst::VecStoreLaneRev { .. } => InstructionSet::VXRS_EXT2,
Inst::DummyUse { .. } => InstructionSet::Base,
+
+ Inst::LabelAddress { .. } => InstructionSet::Base,
}
}
@@ -1006,6 +1008,9 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Ope
Inst::DummyUse { reg } => {
collector.reg_use(reg);
}
+ Inst::LabelAddress { dst, .. } => {
+ collector.reg_def(dst);
+ }
}
}
@@ -3405,6 +3410,10 @@ impl Inst {
let reg = pretty_print_reg(reg);
format!("dummy_use {reg}")
}
+ &Inst::LabelAddress { dst, label } => {
+ let dst = pretty_print_reg(dst.to_reg());
+ format!("label_address {dst}, {label:?}")
+ }
}
}
}
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index 026cf88e69..b86358749c 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -4067,3 +4067,9 @@
(rule (lower (get_return_address))
(load64 (memarg_return_address_offset)))
+
+;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
+ (let ((succ_label MachLabel (block_exn_successor_label block idx)))
+ (s390x_label_address succ_label)))
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 84c65b9a93..242db5a0e3 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -316,6 +316,10 @@
;; A pseudoinstruction that just keeps a value alive.
(DummyUse (reg Reg))
+ ;; A pseudoinstruction that loads the address of a label.
+ (LabelAddress (dst WritableGpr)
+ (label MachLabel))
+
;; An instruction assembled outside of cranelift-codegen.
(External (inst AssemblerInst))))
@@ -3884,6 +3888,14 @@
(_ Unit (emit (MInst.CoffTlsGetAddr name dst tmp))))
dst))
+;;;; Label Addresses ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl x64_label_address (MachLabel) Gpr)
+(rule (x64_label_address label)
+ (let ((dst WritableGpr (temp_writable_gpr))
+ (_ Unit (emit (MInst.LabelAddress dst label))))
+ dst))
+
;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(convert Gpr InstOutput output_gpr)
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 6890c23b52..3f1582ce8d 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1810,6 +1810,11 @@ pub(crate) fn emit(
// Nothing.
}
+ Inst::LabelAddress { dst, label } => {
+ // Emit an LEA with a LabelUse given this label.
+ asm::inst::leaq_rm::new(*dst, Amode::rip_relative(*label)).emit(sink, info, state);
+ }
+
Inst::External { inst } => {
let frame = state.frame_layout();
emit_maybe_shrink(
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index f6ef9eb542..f01dd8781b 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -105,7 +105,8 @@ impl Inst {
| Inst::MachOTlsGetAddr { .. }
| Inst::CoffTlsGetAddr { .. }
| Inst::Unwind { .. }
- | Inst::DummyUse { .. } => true,
+ | Inst::DummyUse { .. }
+ | Inst::LabelAddress { .. } => true,
Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => emit_info.cmpxchg16b(),
@@ -822,6 +823,11 @@ impl PrettyPrint for Inst {
format!("dummy_use {reg}")
}
+ Inst::LabelAddress { dst, label } => {
+ let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
+ format!("label_address {dst}, {label:?}")
+ }
+
Inst::External { inst } => {
format!("{inst}")
}
@@ -1183,6 +1189,10 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_use(reg);
}
+ Inst::LabelAddress { dst, .. } => {
+ collector.reg_def(dst);
+ }
+
Inst::External { inst } => {
inst.visit(&mut external::RegallocVisitor { collector });
}
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 6a5834610e..efbd8fe849 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -5049,6 +5049,12 @@
;; SHUFPS xmm_y, xmm_xmp, 0x88
(x64_shufps dst zeros 0x88)))
+;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))
+ (let ((succ_label MachLabel (block_exn_successor_label block idx)))
+ (x64_label_address succ_label)))
+
;; Rules for `nop` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (nop))
diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs
index 3c593677c9..75a0c8e3d2 100644
--- a/cranelift/codegen/src/isa/x64/pcc.rs
+++ b/cranelift/codegen/src/isa/x64/pcc.rs
@@ -209,6 +209,8 @@ pub(crate) fn check(
Inst::StackSwitchBasic { .. } => Err(PccError::UnimplementedInst),
+ Inst::LabelAddress { .. } => Err(PccError::UnimplementedInst),
+
Inst::External { .. } => Ok(()), // TODO: unsure what to do about this!
}
}
diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs
index 812f341dda..e41f1fa4f1 100644
--- a/cranelift/codegen/src/machinst/blockorder.rs
+++ b/cranelift/codegen/src/machinst/blockorder.rs
@@ -79,6 +79,8 @@ pub struct BlockLoweringOrder {
/// Ranges in `lowered_succ_indices` giving the successor lists for each lowered
/// block. Indexed by lowering-order index (`BlockIndex`).
lowered_succ_ranges: Vec<(Option<Inst>, std::ops::Range<usize>)>,
+ /// BlockIndex for each original Block.
+ blockindex_by_block: SecondaryMap<Block, BlockIndex>,
/// Cold blocks. These blocks are not reordered in the
/// `lowered_order` above; the lowered order must respect RPO
/// (uses after defs) in order for lowering to be
@@ -197,9 +199,11 @@ impl BlockLoweringOrder {
// lowering order, identifying critical edges to split along the way.
let mut lowered_order = Vec::new();
-
+ let mut blockindex_by_block = SecondaryMap::with_default(BlockIndex::invalid());
for &block in domtree.cfg_rpo() {
+ let idx = BlockIndex::new(lowered_order.len());
lowered_order.push(LoweredBlock::Orig { block });
+ blockindex_by_block[block] = idx;
if block_out_count[block] > 1 {
let range = block_succ_range[block].clone();
@@ -296,6 +300,7 @@ impl BlockLoweringOrder {
lowered_order,
lowered_succ_indices,
lowered_succ_ranges,
+ blockindex_by_block,
cold_blocks,
indirect_branch_targets,
};
@@ -309,6 +314,15 @@ impl BlockLoweringOrder {
&self.lowered_order[..]
}
+ /// Get the BlockIndex, if any, for a given Block.
+ ///
+ /// The result will be `None` if the given Block is unreachable
+ /// (and thus does not appear in the lowered order).
+ pub fn lowered_index_for_block(&self, block: Block) -> Option<BlockIndex> {
+ let idx = self.blockindex_by_block[block];
+ if idx.is_valid() { Some(idx) } else { None }
+ }
+
/// Get the successor indices for a lowered block.
pub fn succ_indices(&self, block: BlockIndex) -> (Option<Inst>, &[BlockIndex]) {
let (opt_inst, range) = &self.lowered_succ_ranges[block.index()];
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index 36421a7c43..5f2f1b24c2 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -754,6 +754,14 @@ macro_rules! isle_lower_prelude_methods {
fn value_is_unused(&mut self, val: Value) -> bool {
self.lower_ctx.value_is_unused(val)
}
+
+ fn block_exn_successor_label(&mut self, block: &Block, exn_succ: u64) -> MachLabel {
+ // The first N successors are the exceptional edges, and
+ // the normal return is last; so the `exn_succ`'th
+ // exceptional edge is just the `exn_succ`'th edge overall.
+ let succ = usize::try_from(exn_succ).unwrap();
+ self.lower_ctx.block_successor_label(*block, succ)
+ }
};
}
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index 44ebbf71af..90007f25f9 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -1201,6 +1201,20 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
_ => false,
}
}
+
+ pub fn block_successor_label(&self, block: Block, succ: usize) -> MachLabel {
+ trace!("block_successor_label: block {block} succ {succ}");
+ let lowered = self
+ .vcode
+ .block_order()
+ .lowered_index_for_block(block)
+ .expect("Unreachable block");
+ trace!(" -> lowered block {lowered:?}");
+ let (_, succs) = self.vcode.block_order().succ_indices(lowered);
+ trace!(" -> succs {succs:?}");
+ let succ_block = *succs.get(succ).expect("Successor index out of range");
+ MachLabel::from_block(succ_block)
+ }
}
/// Pre-analysis: compute `value_ir_uses`. See comment on
diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isle
index 069594b940..6bbd3624f4 100644
--- a/cranelift/codegen/src/prelude_lower.isle
+++ b/cranelift/codegen/src/prelude_lower.isle
@@ -381,6 +381,10 @@
(decl sge (Value Value) Value)
(extractor (sge x y) (icmp (IntCC.SignedGreaterThanOrEqual) x y))
+;; Get the MachLabel for a particular CLIF block's indexed exception-handling successor.
+(decl block_exn_successor_label (Block u64) MachLabel)
+(extern constructor block_exn_successor_label block_exn_successor_label)
+
;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Emit an instruction.
diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs
index b07bab2626..b91f1bcc04 100644
--- a/cranelift/codegen/src/verifier/mod.rs
+++ b/cranelift/codegen/src/verifier/mod.rs
@@ -729,6 +729,11 @@ impl<'a> Verifier<'a> {
self.verify_constant_size(inst, opcode, constant_handle, errors)?;
}
+ ExceptionHandlerAddress { block, imm, .. } => {
+ self.verify_block(inst, block, errors)?;
+ self.verify_try_call_handler_index(inst, block, imm.into(), errors)?;
+ }
+
// Exhaustive list so we can't forget to add new formats
AtomicCas { .. }
| AtomicRmw { .. }
@@ -1972,6 +1977,54 @@ impl<'a> Verifier<'a> {
if errors.has_error() { Err(()) } else { Ok(()) }
}
+ fn verify_try_call_handler_index(
+ &self,
+ inst: Inst,
+ block: Block,
+ index_imm: i64,
+ errors: &mut VerifierErrors,
+ ) -> VerifierStepResult {
+ if index_imm < 0 {
+ return errors.fatal((
+ inst,
+ format!("exception handler index {index_imm} cannot be negative"),
+ ));
+ }
+ let Ok(index) = usize::try_from(index_imm) else {
+ return errors.fatal((
+ inst,
+ format!("exception handler index {index_imm} is out-of-range"),
+ ));
+ };
+ let Some(terminator) = self.func.layout.last_inst(block) else {
+ return errors.fatal((
+ inst,
+ format!("referenced block {block} does not have a terminator"),
+ ));
+ };
+ let Some(et) = self.func.dfg.insts[terminator].exception_table() else {
+ return errors.fatal((
+ inst,
+ format!("referenced block {block} does not end in a try_call"),
+ ));
+ };
+
+ let etd = &self.func.dfg.exception_tables[et];
+ // The exception table's out-edges consist of all exceptional
+ // edges first, followed by the normal return last. For N
+ // out-edges, there are N-1 exception handlers that can be
+ // selected.
+ let num_exceptional_edges = etd.all_branches().len() - 1;
+ if index >= num_exceptional_edges {
+ return errors.fatal((
+ inst,
+ format!("exception handler index {index_imm} is out-of-range"),
+ ));
+ }
+
+ Ok(())
+ }
+
pub fn run(&self, errors: &mut VerifierErrors) -> VerifierStepResult {
self.verify_global_values(errors)?;
self.verify_memory_types(errors)?;
diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs
index f9747eafcd..d2fbce11bb 100644
--- a/cranelift/codegen/src/write.rs
+++ b/cranelift/codegen/src/write.rs
@@ -537,6 +537,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
} => write!(w, "{} {}, {}{}", flags, args[0], args[1], offset),
Trap { code, .. } => write!(w, " {code}"),
CondTrap { arg, code, .. } => write!(w, " {arg}, {code}"),
+ ExceptionHandlerAddress { block, imm, .. } => write!(w, " {block}, {imm}"),
}?;
let mut sep = " ; ";
diff --git a/cranelift/filetests/filetests/isa/aarch64/exception-handler-address.clif b/cranelift/filetests/filetests/isa/aarch64/exception-handler-address.clif
new file mode 100644
index 0000000000..b401bdec55
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/exception-handler-address.clif
@@ -0,0 +1,311 @@
+test compile precise-output
+target aarch64
+
+;; Index 0 on get_exception_handler_address selects index 0 in the
+;; exceptional edges, i.e., the one to `block3`. Note the lowering
+;; order reverses block3/block4 when reading the `label_address` inst;
+;; compare to the exception table.
+function %f0() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 0
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ context v1, tag0: block3(exn1), default: block4() ]
+
+ block2:
+ return v1
+
+ block3(v2: i64):
+ return v2
+
+ block4:
+ return v1
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; stp x19, x20, [sp, #-16]!
+; stp d14, d15, [sp, #-16]!
+; stp d12, d13, [sp, #-16]!
+; stp d10, d11, [sp, #-16]!
+; stp d8, d9, [sp, #-16]!
+; sub sp, sp, #16
+; block0:
+; label_address x4, MachLabel(4)
+; str x4, [sp]
+; b label1
+; block1:
+; load_ext_name_far x5, TestCase(%g)+0
+; blr x5; b MachLabel(2); catch [context stack0, tag0: MachLabel(4), default: MachLabel(3)]
+; block2:
+; ldr x0, [sp]
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+; block3:
+; ldr x0, [sp]
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+; block4:
+; mov x0, x1
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; stp x19, x20, [sp, #-0x10]!
+; stp d14, d15, [sp, #-0x10]!
+; stp d12, d13, [sp, #-0x10]!
+; stp d10, d11, [sp, #-0x10]!
+; stp d8, d9, [sp, #-0x10]!
+; sub sp, sp, #0x10
+; block1: ; offset 0x30
+; adr x4, #0xb4
+; stur x4, [sp]
+; block2: ; offset 0x38
+; ldr x5, #0x40
+; b #0x48
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; blr x5
+; block3: ; offset 0x4c
+; ldur x0, [sp]
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+; block4: ; offset 0x80
+; ldur x0, [sp]
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+; block5: ; offset 0xb4
+; mov x0, x1
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
+;; Index 1 on get_exception_handler_address selects index 1 in the
+;; exceptional edges, i.e., the one to `block4`. Note the lowering
+;; order reverses block3/block4 when reading the `label_address` inst;
+;; compare to the exception table.
+function %f0() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 1
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ context v1, tag0: block3(exn1), default: block4() ]
+
+ block2:
+ return v1
+
+ block3(v2: i64):
+ return v2
+
+ block4:
+ return v1
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; stp x19, x20, [sp, #-16]!
+; stp d14, d15, [sp, #-16]!
+; stp d12, d13, [sp, #-16]!
+; stp d10, d11, [sp, #-16]!
+; stp d8, d9, [sp, #-16]!
+; sub sp, sp, #16
+; block0:
+; label_address x4, MachLabel(3)
+; str x4, [sp]
+; b label1
+; block1:
+; load_ext_name_far x5, TestCase(%g)+0
+; blr x5; b MachLabel(2); catch [context stack0, tag0: MachLabel(4), default: MachLabel(3)]
+; block2:
+; ldr x0, [sp]
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+; block3:
+; ldr x0, [sp]
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+; block4:
+; mov x0, x1
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; stp x19, x20, [sp, #-0x10]!
+; stp d14, d15, [sp, #-0x10]!
+; stp d12, d13, [sp, #-0x10]!
+; stp d10, d11, [sp, #-0x10]!
+; stp d8, d9, [sp, #-0x10]!
+; sub sp, sp, #0x10
+; block1: ; offset 0x30
+; adr x4, #0x80
+; stur x4, [sp]
+; block2: ; offset 0x38
+; ldr x5, #0x40
+; b #0x48
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; blr x5
+; block3: ; offset 0x4c
+; ldur x0, [sp]
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+; block4: ; offset 0x80
+; ldur x0, [sp]
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+; block5: ; offset 0xb4
+; mov x0, x1
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif
index 91c029e1cd..f46404987f 100644
--- a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif
@@ -397,3 +397,122 @@ function %f4(i64, i32) -> i32, f32, f64 {
; ldp x29, x30, [sp], #0x10
; ret
+function %f5() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 0
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; stp x19, x20, [sp, #-16]!
+; stp d14, d15, [sp, #-16]!
+; stp d12, d13, [sp, #-16]!
+; stp d10, d11, [sp, #-16]!
+; stp d8, d9, [sp, #-16]!
+; sub sp, sp, #16
+; block0:
+; label_address x0, MachLabel(3)
+; str x0, [sp]
+; b label1
+; block1:
+; load_ext_name_far x4, TestCase(%g)+0
+; blr x4; b MachLabel(2); catch [default: MachLabel(3)]
+; block2:
+; ldr x0, [sp]
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+; block3:
+; ldr x0, [sp]
+; add sp, sp, #16
+; ldp d8, d9, [sp], #16
+; ldp d10, d11, [sp], #16
+; ldp d12, d13, [sp], #16
+; ldp d14, d15, [sp], #16
+; ldp x19, x20, [sp], #16
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; stp x19, x20, [sp, #-0x10]!
+; stp d14, d15, [sp, #-0x10]!
+; stp d12, d13, [sp, #-0x10]!
+; stp d10, d11, [sp, #-0x10]!
+; stp d8, d9, [sp, #-0x10]!
+; sub sp, sp, #0x10
+; block1: ; offset 0x30
+; adr x0, #0x80
+; stur x0, [sp]
+; block2: ; offset 0x38
+; ldr x4, #0x40
+; b #0x48
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; blr x4
+; block3: ; offset 0x4c
+; ldur x0, [sp]
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+; block4: ; offset 0x80
+; ldur x0, [sp]
+; add sp, sp, #0x10
+; ldp d8, d9, [sp], #0x10
+; ldp d10, d11, [sp], #0x10
+; ldp d12, d13, [sp], #0x10
+; ldp d14, d15, [sp], #0x10
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif
index 1087aefc8d..3d701be051 100644
--- a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif
+++ b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif
@@ -182,3 +182,50 @@ function %f4(i32, i32) -> i32, f32, f64 {
; pop_frame_restore 160, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
; ret
+function %f5() -> i32 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i32 block1, 0
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+; VCode:
+; push_frame_save 144, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0}
+; block0:
+; label_address x0, MachLabel(3)
+; xstore64 Slot(0), x0 // flags = notrap aligned
+; jump label1
+; block1:
+; indirect_call_host CallInfo { dest: TestCase(%g), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I32) }], clobbers: PRegSet { bits: [4294967292, 4294967295, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(2), exception_handlers: [Default(MachLabel(3))] }) }; jump MachLabel(2); catch [default: MachLabel(3)]
+; block2:
+; x0 = xload64 Slot(0) // flags = notrap aligned
+; pop_frame_restore 144, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0}
+; ret
+; block3:
+; x0 = xload64 Slot(0) // flags = notrap aligned
+; pop_frame_restore 144, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0}
+; ret
+;
+; Disassembled:
+; push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+; xpcadd x0, 0x20 // target = 0x25
+; xstore64le_o32 sp, 0, x0
+; call_indirect_host 0
+; xload64le_o32 x0, sp, 0
+; pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+; ret
+; xload64le_o32 x0, sp, 0
+; pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif
index ca21591c98..60cb41cf13 100644
--- a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif
+++ b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif
@@ -184,3 +184,50 @@ function %f4(i64, i32) -> i32, f32, f64 {
; pop_frame_restore 160, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
; ret
+function %f5() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 0
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+; VCode:
+; push_frame_save 144, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0}
+; block0:
+; label_address x0, MachLabel(3)
+; xstore64 Slot(0), x0 // flags = notrap aligned
+; jump label1
+; block1:
+; indirect_call_host CallInfo { dest: TestCase(%g), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }], clobbers: PRegSet { bits: [4294967292, 4294967295, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(2), exception_handlers: [Default(MachLabel(3))] }) }; jump MachLabel(2); catch [default: MachLabel(3)]
+; block2:
+; x0 = xload64 Slot(0) // flags = notrap aligned
+; pop_frame_restore 144, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0}
+; ret
+; block3:
+; x0 = xload64 Slot(0) // flags = notrap aligned
+; pop_frame_restore 144, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0}
+; ret
+;
+; Disassembled:
+; push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+; xpcadd x0, 0x20 // target = 0x25
+; xstore64le_o32 sp, 0, x0
+; call_indirect_host 0
+; xload64le_o32 x0, sp, 0
+; pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+; ret
+; xload64le_o32 x0, sp, 0
+; pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif
index b4b172ee8d..859b01d39a 100644
--- a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif
@@ -696,3 +696,220 @@ function %f4(i64, i32) -> i32, f32, f64 {
; addi sp, sp, 0x10
; ret
+function %f5() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 0
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+; VCode:
+; addi sp,sp,-16
+; sd ra,8(sp)
+; sd fp,0(sp)
+; mv fp,sp
+; addi sp,sp,-208
+; sd fp,200(sp)
+; sd s1,192(sp)
+; sd s2,184(sp)
+; sd s3,176(sp)
+; sd s4,168(sp)
+; sd s5,160(sp)
+; sd s6,152(sp)
+; sd s7,144(sp)
+; sd s8,136(sp)
+; sd s9,128(sp)
+; sd s10,120(sp)
+; sd s11,112(sp)
+; fsd fs0,104(sp)
+; fsd fs2,96(sp)
+; fsd fs3,88(sp)
+; fsd fs4,80(sp)
+; fsd fs5,72(sp)
+; fsd fs6,64(sp)
+; fsd fs7,56(sp)
+; fsd fs8,48(sp)
+; fsd fs9,40(sp)
+; fsd fs10,32(sp)
+; fsd fs11,24(sp)
+; block0:
+; label_address a0, MachLabel(3)
+; sd a0,0(slot)
+; j label1
+; block1:
+; load_ext_name_far a4,%g+0
+; callind a4; j MachLabel(2); catch [default: MachLabel(3)]
+; block2:
+; ld a0,0(slot)
+; ld fp,200(sp)
+; ld s1,192(sp)
+; ld s2,184(sp)
+; ld s3,176(sp)
+; ld s4,168(sp)
+; ld s5,160(sp)
+; ld s6,152(sp)
+; ld s7,144(sp)
+; ld s8,136(sp)
+; ld s9,128(sp)
+; ld s10,120(sp)
+; ld s11,112(sp)
+; fld fs0,104(sp)
+; fld fs2,96(sp)
+; fld fs3,88(sp)
+; fld fs4,80(sp)
+; fld fs5,72(sp)
+; fld fs6,64(sp)
+; fld fs7,56(sp)
+; fld fs8,48(sp)
+; fld fs9,40(sp)
+; fld fs10,32(sp)
+; fld fs11,24(sp)
+; addi sp,sp,208
+; ld ra,8(sp)
+; ld fp,0(sp)
+; addi sp,sp,16
+; ret
+; block3:
+; ld a0,0(slot)
+; ld fp,200(sp)
+; ld s1,192(sp)
+; ld s2,184(sp)
+; ld s3,176(sp)
+; ld s4,168(sp)
+; ld s5,160(sp)
+; ld s6,152(sp)
+; ld s7,144(sp)
+; ld s8,136(sp)
+; ld s9,128(sp)
+; ld s10,120(sp)
+; ld s11,112(sp)
+; fld fs0,104(sp)
+; fld fs2,96(sp)
+; fld fs3,88(sp)
+; fld fs4,80(sp)
+; fld fs5,72(sp)
+; fld fs6,64(sp)
+; fld fs7,56(sp)
+; fld fs8,48(sp)
+; fld fs9,40(sp)
+; fld fs10,32(sp)
+; fld fs11,24(sp)
+; addi sp,sp,208
+; ld ra,8(sp)
+; ld fp,0(sp)
+; addi sp,sp,16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; sd s0, 0(sp)
+; mv s0, sp
+; addi sp, sp, -0xd0
+; sd s0, 0xc8(sp)
+; sd s1, 0xc0(sp)
+; sd s2, 0xb8(sp)
+; sd s3, 0xb0(sp)
+; sd s4, 0xa8(sp)
+; sd s5, 0xa0(sp)
+; sd s6, 0x98(sp)
+; sd s7, 0x90(sp)
+; sd s8, 0x88(sp)
+; sd s9, 0x80(sp)
+; sd s10, 0x78(sp)
+; sd s11, 0x70(sp)
+; fsd fs0, 0x68(sp)
+; fsd fs2, 0x60(sp)
+; fsd fs3, 0x58(sp)
+; fsd fs4, 0x50(sp)
+; fsd fs5, 0x48(sp)
+; fsd fs6, 0x40(sp)
+; fsd fs7, 0x38(sp)
+; fsd fs8, 0x30(sp)
+; fsd fs9, 0x28(sp)
+; fsd fs10, 0x20(sp)
+; fsd fs11, 0x18(sp)
+; block1: ; offset 0x70
+; auipc a0, 0
+; addi a0, a0, 0x98
+; sd a0, 0(sp)
+; block2: ; offset 0x7c
+; auipc a4, 0
+; ld a4, 0xc(a4)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; jalr a4
+; block3: ; offset 0x94
+; ld a0, 0(sp)
+; ld s0, 0xc8(sp)
+; ld s1, 0xc0(sp)
+; ld s2, 0xb8(sp)
+; ld s3, 0xb0(sp)
+; ld s4, 0xa8(sp)
+; ld s5, 0xa0(sp)
+; ld s6, 0x98(sp)
+; ld s7, 0x90(sp)
+; ld s8, 0x88(sp)
+; ld s9, 0x80(sp)
+; ld s10, 0x78(sp)
+; ld s11, 0x70(sp)
+; fld fs0, 0x68(sp)
+; fld fs2, 0x60(sp)
+; fld fs3, 0x58(sp)
+; fld fs4, 0x50(sp)
+; fld fs5, 0x48(sp)
+; fld fs6, 0x40(sp)
+; fld fs7, 0x38(sp)
+; fld fs8, 0x30(sp)
+; fld fs9, 0x28(sp)
+; fld fs10, 0x20(sp)
+; fld fs11, 0x18(sp)
+; addi sp, sp, 0xd0
+; ld ra, 8(sp)
+; ld s0, 0(sp)
+; addi sp, sp, 0x10
+; ret
+; block4: ; offset 0x108
+; ld a0, 0(sp)
+; ld s0, 0xc8(sp)
+; ld s1, 0xc0(sp)
+; ld s2, 0xb8(sp)
+; ld s3, 0xb0(sp)
+; ld s4, 0xa8(sp)
+; ld s5, 0xa0(sp)
+; ld s6, 0x98(sp)
+; ld s7, 0x90(sp)
+; ld s8, 0x88(sp)
+; ld s9, 0x80(sp)
+; ld s10, 0x78(sp)
+; ld s11, 0x70(sp)
+; fld fs0, 0x68(sp)
+; fld fs2, 0x60(sp)
+; fld fs3, 0x58(sp)
+; fld fs4, 0x50(sp)
+; fld fs5, 0x48(sp)
+; fld fs6, 0x40(sp)
+; fld fs7, 0x38(sp)
+; fld fs8, 0x30(sp)
+; fld fs9, 0x28(sp)
+; fld fs10, 0x20(sp)
+; fld fs11, 0x18(sp)
+; addi sp, sp, 0xd0
+; ld ra, 8(sp)
+; ld s0, 0(sp)
+; addi sp, sp, 0x10
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/s390x/exceptions.clif b/cranelift/filetests/filetests/isa/s390x/exceptions.clif
index b67010a707..17cc9e96b5 100644
--- a/cranelift/filetests/filetests/isa/s390x/exceptions.clif
+++ b/cranelift/filetests/filetests/isa/s390x/exceptions.clif
@@ -396,3 +396,112 @@ function %f4(i64, i32) -> i32, f32, f64 {
; .byte 0x00, 0x00
; .byte 0x00, 0x00
+function %f5() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 0
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+; VCode:
+; stmg %r6, %r15, 48(%r15)
+; aghi %r15, -232
+; std %f8, 168(%r15)
+; std %f9, 176(%r15)
+; std %f10, 184(%r15)
+; std %f11, 192(%r15)
+; std %f12, 200(%r15)
+; std %f13, 208(%r15)
+; std %f14, 216(%r15)
+; std %f15, 224(%r15)
+; block0:
+; label_address %r2, MachLabel(3)
+; stg %r2, 160(%r15)
+; jg label1
+; block1:
+; bras %r1, 12 ; data %g + 0 ; lg %r2, 0(%r1)
+; basr %r14, %r2; jg MachLabel(2); catch [default: MachLabel(3)]
+; block2:
+; lg %r2, 160(%r15)
+; ld %f8, 168(%r15)
+; ld %f9, 176(%r15)
+; ld %f10, 184(%r15)
+; ld %f11, 192(%r15)
+; ld %f12, 200(%r15)
+; ld %f13, 208(%r15)
+; ld %f14, 216(%r15)
+; ld %f15, 224(%r15)
+; lmg %r6, %r15, 280(%r15)
+; br %r14
+; block3:
+; lg %r2, 160(%r15)
+; ld %f8, 168(%r15)
+; ld %f9, 176(%r15)
+; ld %f10, 184(%r15)
+; ld %f11, 192(%r15)
+; ld %f12, 200(%r15)
+; ld %f13, 208(%r15)
+; ld %f14, 216(%r15)
+; ld %f15, 224(%r15)
+; lmg %r6, %r15, 280(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r6, %r15, 0x30(%r15)
+; aghi %r15, -0xe8
+; std %f8, 0xa8(%r15)
+; std %f9, 0xb0(%r15)
+; std %f10, 0xb8(%r15)
+; std %f11, 0xc0(%r15)
+; std %f12, 0xc8(%r15)
+; std %f13, 0xd0(%r15)
+; std %f14, 0xd8(%r15)
+; std %f15, 0xe0(%r15)
+; block1: ; offset 0x2a
+; larl %r2, 0x78
+; stg %r2, 0xa0(%r15)
+; block2: ; offset 0x36
+; bras %r1, 0x42
+; .byte 0x00, 0x00 ; reloc_external Abs8 %g 0
+; .byte 0x00, 0x00
+; .byte 0x00, 0x00
+; .byte 0x00, 0x00
+; lg %r2, 0(%r1)
+; basr %r14, %r2
+; block3: ; offset 0x4a
+; lg %r2, 0xa0(%r15)
+; ld %f8, 0xa8(%r15)
+; ld %f9, 0xb0(%r15)
+; ld %f10, 0xb8(%r15)
+; ld %f11, 0xc0(%r15)
+; ld %f12, 0xc8(%r15)
+; ld %f13, 0xd0(%r15)
+; ld %f14, 0xd8(%r15)
+; ld %f15, 0xe0(%r15)
+; lmg %r6, %r15, 0x118(%r15)
+; br %r14
+; block4: ; offset 0x78
+; lg %r2, 0xa0(%r15)
+; ld %f8, 0xa8(%r15)
+; ld %f9, 0xb0(%r15)
+; ld %f10, 0xb8(%r15)
+; ld %f11, 0xc0(%r15)
+; ld %f12, 0xc8(%r15)
+; ld %f13, 0xd0(%r15)
+; ld %f14, 0xd8(%r15)
+; ld %f15, 0xe0(%r15)
+; lmg %r6, %r15, 0x118(%r15)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/x64/exceptions.clif b/cranelift/filetests/filetests/isa/x64/exceptions.clif
index a83b34e69e..1afdc69052 100644
--- a/cranelift/filetests/filetests/isa/x64/exceptions.clif
+++ b/cranelift/filetests/filetests/isa/x64/exceptions.clif
@@ -539,3 +539,100 @@ function %f4(i64, i32) -> i32, f32, f64 {
; popq %rbp
; retq
+
+function %f5() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 0
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x40, %rsp
+; movq %rbx, 0x10(%rsp)
+; movq %r12, 0x18(%rsp)
+; movq %r13, 0x20(%rsp)
+; movq %r14, 0x28(%rsp)
+; movq %r15, 0x30(%rsp)
+; block0:
+; label_address %rax, MachLabel(3)
+; movq %rax, <offset:1>+(%rsp)
+; jmp label1
+; block1:
+; load_ext_name %g+0, %rdx
+; call *%rdx; jmp MachLabel(2); catch [default: MachLabel(3)]
+; block2:
+; movq <offset:1>+(%rsp), %rax
+; movq 0x10(%rsp), %rbx
+; movq 0x18(%rsp), %r12
+; movq 0x20(%rsp), %r13
+; movq 0x28(%rsp), %r14
+; movq 0x30(%rsp), %r15
+; addq $0x40, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3:
+; movq <offset:1>+(%rsp), %rax
+; movq 0x10(%rsp), %rbx
+; movq 0x18(%rsp), %r12
+; movq 0x20(%rsp), %r13
+; movq 0x28(%rsp), %r14
+; movq 0x30(%rsp), %r15
+; addq $0x40, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x40, %rsp
+; movq %rbx, 0x10(%rsp)
+; movq %r12, 0x18(%rsp)
+; movq %r13, 0x20(%rsp)
+; movq %r14, 0x28(%rsp)
+; movq %r15, 0x30(%rsp)
+; block1: ; offset 0x21
+; leaq 0x36(%rip), %rax
+; movq %rax, (%rsp)
+; block2: ; offset 0x2c
+; movabsq $0, %rdx ; reloc_external Abs8 %g 0
+; callq *%rdx
+; block3: ; offset 0x38
+; movq (%rsp), %rax
+; movq 0x10(%rsp), %rbx
+; movq 0x18(%rsp), %r12
+; movq 0x20(%rsp), %r13
+; movq 0x28(%rsp), %r14
+; movq 0x30(%rsp), %r15
+; addq $0x40, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block4: ; offset 0x5e
+; movq (%rsp), %rax
+; movq 0x10(%rsp), %rbx
+; movq 0x18(%rsp), %r12
+; movq 0x20(%rsp), %r13
+; movq 0x28(%rsp), %r14
+; movq 0x30(%rsp), %r15
+; addq $0x40, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/verifier/exceptions.clif b/cranelift/filetests/filetests/verifier/exceptions.clif
index 6dedb7e34f..f2e97b7740 100644
--- a/cranelift/filetests/filetests/verifier/exceptions.clif
+++ b/cranelift/filetests/filetests/verifier/exceptions.clif
@@ -209,3 +209,99 @@ function %f5() {
block2(v1: i64, v2: i64):
return
}
+
+;; invalid successor index on `get_exception_handler_address`
+function %f6() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 1 ; error: exception handler index 1 is out-of-range
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+;; invalid successor index on `get_exception_handler_address`
+function %f7() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, -1 ; error: exception handler index -1 cannot be negative
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+;; invalid block on `get_exception_handler_address`
+function %f8() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block17, 0 ; error: invalid block reference block17
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+;; invalid block terminator on `get_exception_handler_address`
+function %f8() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block2, 0 ; error: referenced block block2 does not end in a try_call
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), [ default: block3() ]
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
+
+
+;; invalid block terminator on `get_exception_handler_address`
+function %f9() -> i64 {
+ sig0 = () tail
+ fn0 = %g() tail
+
+ block0:
+ v1 = get_exception_handler_address.i64 block1, 0 ; error: exception handler index 0 is out-of-range
+ jump block1
+
+ block1:
+ try_call fn0(), sig0, block2(), []
+
+ block2:
+ return v1
+
+ block3:
+ return v1
+}
diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs
index 6c01851740..325034e084 100644
--- a/cranelift/fuzzgen/src/function_generator.rs
+++ b/cranelift/fuzzgen/src/function_generator.rs
@@ -1104,6 +1104,7 @@ fn inserter_for_format(fmt: InstructionFormat) -> OpcodeInserter {
InstructionFormat::UnaryIeee32 => insert_const,
InstructionFormat::UnaryIeee64 => insert_const,
InstructionFormat::UnaryImm => insert_const,
+ InstructionFormat::ExceptionHandlerAddress => insert_const,
InstructionFormat::BranchTable
| InstructionFormat::Brif
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index 3bf387bd43..ac4bee433d 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -1319,6 +1319,8 @@ where
Opcode::TryCall => unimplemented!("TryCall"),
Opcode::TryCallIndirect => unimplemented!("TryCallIndirect"),
+
+ Opcode::GetExceptionHandlerAddress => unimplemented!("GetExceptionHandlerAddress"),
})
}
diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs
index 0a9675e6a8..080476a44f 100644
--- a/cranelift/reader/src/parser.rs
+++ b/cranelift/reader/src/parser.rs
@@ -3347,6 +3347,12 @@ impl<'a> Parser<'a> {
code,
}
}
+ InstructionFormat::ExceptionHandlerAddress => {
+ let block = self.match_block("expected block")?;
+ self.match_token(Token::Comma, "expected ',' between operands")?;
+ let imm = self.match_imm64("expected immediate handler index")?;
+ InstructionData::ExceptionHandlerAddress { opcode, block, imm }
+ }
};
Ok(idata)
}