Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: add opportunistic value defs, use for `uadd_overflow` + `brif` folds. (#14228)

Chris Fallin34dded7f5033
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 5856b27164..17b5bafde9 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -4946,6 +4946,17 @@
(rule 3 (is_nonzero val @ (value_type $I8))
(CondResult.Cond (tst_imm $I32 val (u64_into_imm_logic $I32 255)) (Cond.Ne)))
+;; Use of the flags output of `uadd_overflow` can produce a `Cond` and
+;; give an opportunistic def of the other output.
+(rule 4
+ (is_nonzero
+ (second_result uadd @ (uadd_overflow (ty_32_or_64 ty) x y)))
+ (if-let (first_result sum_value) uadd)
+ (let ((producer ProducesFlags (alu_rrr_with_flags_paired ty x y (ALUOp.AddS))))
+ (CondResult.Cond
+ (produces_flags_opportunistic_def producer sum_value)
+ (Cond.Hs))))
+
(attr emit_icmp (veri chain))
(decl emit_icmp (IntCC Value Value) CondResult)
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index e9806dce21..c6a3e3679b 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -2966,6 +2966,14 @@
;;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; The flags output of a `uadd_overflow` that is not demanded as a materialized
+;; bool (i.e., its only use(s) were folded into consumers such as a `brif` that
+;; consume the flags directly) does not need a `cset`; emit just the `adds`.
+(rule 3 (lower i @ (uadd_overflow (ty_32_or_64 ty) x y))
+ (if-let (second_result flags) i)
+ (if-let false (value_used flags))
+ (output_pair (add ty x y) (value_regs_invalid)))
+
;; For values smaller than a register, we do a normal `add` with both arguments
;; zero extended. We then check if the output is the same as itself zero extended.
(rule 1 (lower (uadd_overflow (fits_in_16 ty) a b))
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 57639f6ff0..be4b4d6464 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -3875,6 +3875,17 @@
(rule 2 (is_nonzero (band _ a @ (value_type (ty_int (fits_in_64 ty))) b))
(is_nonzero_band ty a b))
+;; Using the flags output of `uadd_overflow` can produce a `CC` and give an
+;; opportunistic def of the other output.
+(rule 3
+ (is_nonzero
+ (second_result uadd @ (uadd_overflow (fits_in_64 ty) x y)))
+ (if-let (first_result sum_value) uadd)
+ (let ((producer ProducesFlags (x64_add_with_flags_paired ty x y)))
+ (CondResult.CC
+ (produces_flags_opportunistic_def producer sum_value)
+ (CC.B))))
+
;; Like `is_nonzero` but with additional specializations for compare
;; operators. We break this out from `is_nonzero` because we want to
;; avoid unbounded recursion.
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 8739c6fb6c..cabc44e5a8 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -178,6 +178,14 @@
;;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; The flags output of a `uadd_overflow` that is not demanded as a materialized
+;; bool (i.e., its only use(s) were folded into consumers such as a `brif` that
+;; consume the flags directly) does not need a `setcc`; emit just the `add`.
+(rule 3 (lower i @ (uadd_overflow (fits_in_64 ty) x y))
+ (if-let (second_result flags) i)
+ (if-let false (value_used flags))
+ (output_pair (value_reg (x64_add ty x y)) (value_regs_invalid)))
+
(rule 1 (lower (uadd_overflow _ x y @ (value_type (fits_in_64 ty))))
(construct_overflow_op_alu ty (CC.B) (ProduceFlagsOp.Add) x y))
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index 499b807f72..c05127ff0d 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -114,6 +114,11 @@ macro_rules! isle_lower_prelude_methods {
self.lower_ctx.increment_lowered_uses(val);
}
+ #[inline]
+ fn opportunistic_def(&mut self, val: Value, regs: ValueRegs) {
+ self.lower_ctx.opportunistic_def(val, regs);
+ }
+
#[inline]
fn put_in_reg(&mut self, val: Value) -> Reg {
self.put_in_regs(val).only_reg().unwrap()
@@ -197,6 +202,17 @@ macro_rules! isle_lower_prelude_methods {
self.lower_ctx.dfg().inst_results(inst).first().copied()
}
+ #[inline]
+ fn second_result(&mut self, inst: Inst) -> Option<Value> {
+ self.lower_ctx
+ .dfg()
+ .inst_results(inst)
+ .iter()
+ .skip(1)
+ .next()
+ .copied()
+ }
+
#[inline]
fn inst_data_value(&mut self, inst: Inst) -> (Type, InstructionData) {
let ty = match self.first_result(inst) {
@@ -778,6 +794,10 @@ macro_rules! isle_lower_prelude_methods {
self.lower_ctx.value_is_unused(val)
}
+ fn value_used(&mut self, val: Value) -> bool {
+ self.lower_ctx.value_lowered_used(val)
+ }
+
fn block_exn_successor_label(&mut self, block: &Block, exn_succ: u64) -> MachLabel {
// The first N successors are the exceptional edges, and
// the normal return is last; so the `exn_succ`'th
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index c006c6b640..1413fddb33 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -204,6 +204,22 @@ pub struct Lower<'func, I: VCodeInst> {
/// Actual uses of each SSA value so far, incremented while lowering.
value_lowered_uses: SecondaryMap<Value, u32>,
+ /// "Opportunistic defs" of values: when lowering an instruction that
+ /// incidentally computes another value (e.g., a branch that directly
+ /// consumes the flags of an `uadd_overflow` also computes the sum), we
+ /// record that value here, along with the regs it was computed into and
+ /// the use-count at the time of registration.
+ ///
+ /// When the scan reaches the actual definition of such a value, if its
+ /// use-count has not grown (i.e., no further uses were found while
+ /// scanning up), then the definition can be skipped and the value can be
+ /// aliased to the opportunistically-computed regs instead.
+ ///
+ /// The key is (block, value): the opportunistic def is only usable when
+ /// the actual definition is in the same block as the registration site,
+ /// further up in the scan.
+ opportunistic_defs: FxHashMap<(Block, Value), (ValueRegs<Reg>, u32)>,
+
/// Effectful instructions that have been sunk; they are not codegen'd at
/// their original locations.
inst_sunk: FxHashSet<Inst>,
@@ -498,6 +514,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
side_effect_inst_entry_colors,
value_ir_uses,
value_lowered_uses: SecondaryMap::default(),
+ opportunistic_defs: FxHashMap::default(),
inst_sunk: FxHashSet::default(),
cur_scan_entry_color: None,
cur_inst: None,
@@ -712,6 +729,112 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
.any(|&result| self.value_lowered_uses[result] > 0)
}
+ /// Record an "opportunistic def" of `val` into `regs` at the current scan
+ /// position.
+ ///
+ /// The lowering that is currently being generated (e.g., a branch that
+ /// directly consumes the flags produced by an `uadd_overflow`) also
+ /// computes `val`'s value as a byproduct, and does so in `regs`. If, when
+ /// the scan reaches the actual definition of `val` (which must be in the
+ /// current block, further up), no further uses of `val` are found (i.e.,
+ /// the use-count matches the one recorded here), then that definition can
+ /// be skipped entirely and `val` can be aliased to `regs` instead.
+ ///
+ /// If further uses *are* found, then this opportunistic def is discarded
+ /// and the actual definition is lowered as usual (this is safe because
+ /// the value is still computed by the current lowering, just unused).
+ pub fn opportunistic_def(&mut self, val: Value, regs: ValueRegs<Reg>) {
+ trace!("opportunistic_def: val {val} regs {regs:?}");
+
+ if self.value_lowered_uses[val] == 0 {
+ trace!(" -> no uses so far; ignoring");
+ return;
+ }
+
+ // The actual definition of `val` must be in the same block as the
+ // current scan position, further up. Otherwise the regs computed here
+ // cannot possibly dominate all uses of `val` (and in particular the
+ // use-count check below is meaningless across blocks), so ignore.
+ let cur_block = match self
+ .cur_inst
+ .and_then(|inst| self.f.layout.inst_block(inst))
+ {
+ Some(block) => block,
+ None => {
+ trace!(" -> no current inst/block; ignoring");
+ return;
+ }
+ };
+ let def_block = match self.f.dfg.value_def(val) {
+ ValueDef::Result(src_inst, _) => self.f.layout.inst_block(src_inst),
+ _ => None,
+ };
+ if def_block != Some(cur_block) {
+ trace!(" -> def not in current block; ignoring");
+ return;
+ }
+
+ let uses = self.value_lowered_uses[val];
+ // Note that a later registration for the same (block, value)
+ // overwrites an earlier one: the later one is higher in the block
+ // (closer to the definition), so its defs dominate those of the
+ // earlier one, and it is strictly more useful.
+ self.opportunistic_defs
+ .insert((cur_block, val), (regs, uses));
+ trace!(" -> recorded with {uses} uses so far");
+ }
+
+ /// Attempt to commit to the opportunistic defs recorded for the results
+ /// of `inst` (whose definition is at the current scan position in
+ /// `block`).
+ ///
+ /// Returns `true` if we were able to use the opportunistic defs
+ /// and can skip this lowering.
+ fn try_use_opportunistic_defs(&mut self, block: Block, inst: Inst) -> bool {
+ let results = self.f.dfg.inst_results(inst);
+
+ // To skip the lowering and use the opportunistic defs, every
+ // result of `inst` that has uses must have a registered
+ // opportunistic def whose recorded use-count matches the
+ // current one.
+ for &result in results {
+ if self.value_lowered_uses[result] == 0 {
+ continue;
+ }
+ match self.opportunistic_defs.get(&(block, result)) {
+ Some(&(_, recorded_uses)) if recorded_uses == self.value_lowered_uses[result] => {}
+ _ => {
+ trace!(
+ "opportunistic defs: not committing for inst {inst}: \
+ result {result} has {} uses but no matching opportunistic def",
+ self.value_lowered_uses[result]
+ );
+ return false;
+ }
+ }
+ }
+
+ // Commit: set aliases for every result that has an opportunistic def
+ // (by the check above, this is exactly the set of results with uses),
+ // and clear the entries.
+ for &result in results {
+ if let Some(&(regs, _)) = self.opportunistic_defs.get(&(block, result)) {
+ let dsts = self.value_regs[result];
+ debug_assert_eq!(dsts.len(), regs.len());
+ for (&dst, &src) in dsts.regs().iter().zip(regs.regs().iter()) {
+ trace!(
+ "set vreg alias (opportunistic def): {result:?} = {dst:?}, \
+ lowering = {src:?}"
+ );
+ self.vregs.set_vreg_alias(dst, src);
+ }
+ self.opportunistic_defs.remove(&(block, result));
+ }
+ }
+
+ true
+ }
+
fn lower_clif_block<B: LowerBackend<MInst = I>>(
&mut self,
backend: &B,
@@ -789,6 +912,15 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
// Normal instruction: codegen if the instruction is side-effecting
// or any of its outputs is used.
if must_lower || value_needed {
+ if !has_side_effect && !must_lower && self.try_use_opportunistic_defs(block, inst) {
+ trace!(
+ "lowering: inst {}: {}: using opportunistic defs; skipping",
+ inst,
+ self.f.dfg.display_inst(inst)
+ );
+ continue;
+ }
+
trace!("lowering: inst {}: {}", inst, self.f.dfg.display_inst(inst));
let temp_regs = match backend.lower(self, inst) {
Some(regs) => regs,
@@ -806,16 +938,17 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
}
};
- // The ISLE generated code emits its own registers to define the
- // instruction's lowered values in. However, other instructions
- // that use this SSA value will be lowered assuming that the value
- // is generated into a pre-assigned, different, register.
+ // The ISLE generated code emits its own registers to define
+ // the instruction's lowered values in. However, other
+ // instructions that use this SSA value will be lowered
+ // assuming that the value is generated into a
+ // pre-assigned, different, register.
//
- // To connect the two, we set up "aliases" in the VCodeBuilder
- // that apply when it is building the Operand table for the
- // regalloc to use. These aliases effectively rewrite any use of
- // the pre-assigned register to the register that was returned by
- // the ISLE lowering logic.
+ // To connect the two, we set up "aliases" in the
+ // VCodeBuilder that apply when it is building the Operand
+ // table for the regalloc to use. These aliases effectively
+ // rewrite any use of the pre-assigned register to the
+ // register that was returned by the ISLE lowering logic.
let results = self.f.dfg.inst_results(inst);
debug_assert_eq!(temp_regs.len(), results.len());
for (regs, &result) in temp_regs.iter().zip(results) {
@@ -1221,6 +1354,12 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
}
}
+ /// Does this value still have uses to serve at the current point in the
+ /// lowering scan? If not, a lowering may be elided.
+ pub(crate) fn value_lowered_used(&self, val: Value) -> bool {
+ self.value_lowered_uses[val] > 0
+ }
+
pub fn block_successor_label(&self, block: Block, succ: usize) -> MachLabel {
trace!("block_successor_label: block {block} succ {succ}");
let lowered = self
diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isle
index f4e0ae86cb..1eadf632f9 100644
--- a/cranelift/codegen/src/prelude_lower.isle
+++ b/cranelift/codegen/src/prelude_lower.isle
@@ -128,6 +128,14 @@
(decl mark_value_used (Value) Unit)
(extern constructor mark_value_used mark_value_used)
+;; Record an "opportunistic def" of `val` into `regs` at the current
+;; scan position: the lowering currently being generated computes a
+;; value as a byproduct into the given reg(s). If this is sufficient
+;; to satisfy all uses, the lowering at the original def site may be
+;; elided.
+(decl opportunistic_def (Value ValueRegs) Unit)
+(extern constructor opportunistic_def opportunistic_def)
+
;; Put the given value into a register.
;;
;; Asserts that the value fits into a single register, and doesn't require
@@ -270,10 +278,20 @@
(decl pure value_is_unused (Value) bool)
(extern constructor value_is_unused value_is_unused)
+;; Returns whether the given value still has uses to serve in the
+;; lowering scan. If false, production of the given value may be
+;; skipped by the lowered instruction(s).
+(decl pure value_used (Value) bool)
+(extern constructor value_used value_used)
+
;; Extract the first result value of the given instruction.
(decl first_result (Value) Inst)
(extern extractor first_result first_result)
+;; Extract the second result value of the given instruction.
+(decl second_result (Value) Inst)
+(extern extractor second_result second_result)
+
;; Extract the `InstructionData` for an `Inst`.
(decl inst_data_value (Type InstructionData) Inst)
(extern extractor infallible inst_data_value inst_data_value)
@@ -524,7 +542,24 @@
;; Not directly combinable with a ConsumesFlags;
;; used in s390x and unwrapped directly by `trapif`.
(ProducesFlagsReturnsReg (inst MInst) (result Reg))
- (ProducesFlagsReturnsResultWithConsumer (inst MInst) (result Reg))))
+ (ProducesFlagsReturnsResultWithConsumer (inst MInst) (result Reg))
+ ;; Like `ProducesFlagsReturnsResultWithConsumer`, but the
+ ;; result register is not exposed as a normal result: when
+ ;; the producer is emitted (via `with_flags` or
+ ;; `with_flags_side_effect`), an "opportunistic def" of
+ ;; `value` is registered in `reg` instead. This allows a
+ ;; lowering that consumes only the flags (e.g., a branch,
+ ;; trap, or select on the overflow-flag output of a
+ ;; `uadd_overflow`) to incidentally define the other
+ ;; output (the sum) so that its own lowering can be
+ ;; skipped.
+ ;;
+ ;; The registration is deliberately deferred until
+ ;; emission time, because the consumer may materialize
+ ;; uses of `value` (e.g., as its own operands) after the
+ ;; producer is constructed, and we want these to use the
+ ;; opportunistic def if they can, as well.
+ (ProducesFlagsOpportunisticDef (inst MInst) (reg Reg) (value Value))))
(spec (ProducesFlags.ProducesFlagsReturnsResultWithConsumer inst reg)
(provide
@@ -532,6 +567,11 @@
(= (:result result) reg)
(:has_result result)))
+(spec (ProducesFlags.ProducesFlagsOpportunisticDef inst reg value)
+ (provide
+ (= (:flags result) (:flags_out inst))
+ (not (:has_result result))))
+
(spec (ProducesFlags.ProducesFlagsSideEffect inst)
(provide
(= (:flags result) (:flags_out inst))
@@ -582,7 +622,12 @@
(inst2 MInst)
(inst3 MInst)
(inst4 MInst)
- (result ValueRegs))))
+ (result ValueRegs))
+ (ConsumesFlagsNop)))
+
+(spec (ConsumesFlags.ConsumesFlagsNop)
+ (provide
+ (not (:has_result result))))
(spec (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer inst reg)
(provide
@@ -614,6 +659,14 @@
(rule (produces_flags_ignore (ProducesFlags.ProducesFlagsReturnsResultWithConsumer inst _))
(ProducesFlags.ProducesFlagsSideEffect inst))
+;; Wrap a result-producing producer so that, when it is emitted, an
+;; opportunistic def of `value` is registered in its result register (see
+;; the `ProducesFlagsOpportunisticDef` variant).
+(decl produces_flags_opportunistic_def (ProducesFlags Value) ProducesFlags)
+(rule (produces_flags_opportunistic_def
+ (ProducesFlags.ProducesFlagsReturnsResultWithConsumer inst reg) value)
+ (ProducesFlags.ProducesFlagsOpportunisticDef inst reg value))
+
;; Helper for combining two flags-consumer instructions that return a
;; single Reg, giving a ConsumesFlags that returns both values in a
;; ValueRegs.
@@ -691,6 +744,19 @@
(_ Unit (emit consumer_inst)))
(value_reg producer_result)))
+;; A flag-producer whose flags are not consumed by anything: emit it for its
+;; result alone, ignoring the flags.
+(rule with_flags_producer_result_only_noop
+ (with_flags (ProducesFlags.ProducesFlagsReturnsResultWithConsumer producer_inst producer_result)
+ (ConsumesFlags.ConsumesFlagsNop))
+ (let ((_x Unit (emit producer_inst)))
+ (value_reg producer_result)))
+(rule with_flags_producer_reg_noop
+ (with_flags (ProducesFlags.ProducesFlagsReturnsReg producer_inst producer_result)
+ (ConsumesFlags.ConsumesFlagsNop))
+ (let ((_x Unit (emit producer_inst)))
+ (value_reg producer_result)))
+
(rule with_flags_consumer_reg
(with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
(ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))
@@ -698,6 +764,18 @@
(_y Unit (emit consumer_inst)))
(value_reg consumer_result)))
+;; A producer that registers an opportunistic def of another value in
+;; `producer_result` when it is emitted: record the def only after the
+;; emission has been set up, so that a failed match does not leave a dangling
+;; registration.
+(rule with_flags_opportunistic_def_consumer_reg
+ (with_flags (ProducesFlags.ProducesFlagsOpportunisticDef producer_inst producer_result producer_value)
+ (ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))
+ (let ((_x Unit (emit producer_inst))
+ (_y Unit (emit consumer_inst))
+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))
+ (value_reg consumer_result)))
+
(rule with_flags_consumer_twice
(with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs consumer_inst_1
@@ -711,6 +789,20 @@
(_z Unit (emit consumer_inst_2)))
consumer_result))
+(rule with_flags_opportunistic_def_consumer_twice
+ (with_flags (ProducesFlags.ProducesFlagsOpportunisticDef producer_inst producer_result producer_value)
+ (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs consumer_inst_1
+ consumer_inst_2
+ consumer_result))
+ ;; We must emit these instructions in order as the creator of
+ ;; the ConsumesFlags may be relying on dataflow dependencies
+ ;; amongst them.
+ (let ((_x Unit (emit producer_inst))
+ (_y Unit (emit consumer_inst_1))
+ (_z Unit (emit consumer_inst_2))
+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))
+ consumer_result))
+
(rule with_flags_consumer_four
(with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
(ConsumesFlags.ConsumesFlagsFourTimesReturnsValueRegs consumer_inst_1
@@ -728,6 +820,24 @@
(_v Unit (emit consumer_inst_4)))
consumer_result))
+(rule with_flags_opportunistic_def_consumer_four
+ (with_flags (ProducesFlags.ProducesFlagsOpportunisticDef producer_inst producer_result producer_value)
+ (ConsumesFlags.ConsumesFlagsFourTimesReturnsValueRegs consumer_inst_1
+ consumer_inst_2
+ consumer_inst_3
+ consumer_inst_4
+ consumer_result))
+ ;; We must emit these instructions in order as the creator of
+ ;; the ConsumesFlags may be relying on dataflow dependencies
+ ;; amongst them.
+ (let ((_x Unit (emit producer_inst))
+ (_y Unit (emit consumer_inst_1))
+ (_z Unit (emit consumer_inst_2))
+ (_w Unit (emit consumer_inst_3))
+ (_v Unit (emit consumer_inst_4))
+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))
+ consumer_result))
+
(rule with_flags_producer_twice_reg
(with_flags (ProducesFlags.ProducesFlagsTwiceSideEffect producer_inst1 producer_inst2)
(ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))
@@ -776,10 +886,27 @@
;; Combine a flags-producing instruction and a flags-consuming instruction that
;; produces no results.
;;
-;; This function handles the following case only:
-;; - ProducesFlagsSideEffect + ConsumesFlagsSideEffect
+;; This function handles the following cases:
+;; - ProducesFlagsSideEffect + ConsumesFlagsSideEffect / ConsumesFlagsSideEffect2
+;; - ProducesFlagsOpportunisticDef + ConsumesFlagsSideEffect / ConsumesFlagsSideEffect2
(decl with_flags_side_effect (ProducesFlags ConsumesFlags) SideEffectNoResult)
+;; A producer that registers an opportunistic def of another value in
+;; `producer_result` when it is emitted (see `with_flags`): record the def
+;; here, when the emission is set up, so that a failed match does not leave a
+;; dangling registration.
+(rule (with_flags_side_effect
+ (ProducesFlags.ProducesFlagsOpportunisticDef producer_inst producer_result producer_value)
+ (ConsumesFlags.ConsumesFlagsSideEffect consumer_inst))
+ (let ((_ Unit (opportunistic_def producer_value (value_reg producer_result))))
+ (SideEffectNoResult.Inst2 producer_inst consumer_inst)))
+
+(rule (with_flags_side_effect
+ (ProducesFlags.ProducesFlagsOpportunisticDef producer_inst producer_result producer_value)
+ (ConsumesFlags.ConsumesFlagsSideEffect2 consumer_inst_1 consumer_inst_2))
+ (let ((_ Unit (opportunistic_def producer_value (value_reg producer_result))))
+ (SideEffectNoResult.Inst3 producer_inst consumer_inst_1 consumer_inst_2)))
+
(rule (with_flags_side_effect
(ProducesFlags.AlreadyExistingFlags)
(ConsumesFlags.ConsumesFlagsSideEffect c))
@@ -855,6 +982,7 @@
(MultiReg.Two (value_regs_get consume_result 0) (value_regs_get consume_result 1))))
+
;; ProducesFlags.ReturnsReg + ConsumesAndProducesFlags.SideEffect with all possible ConsumeFlags options
(rule (with_flags_chained (ProducesFlags.ProducesFlagsReturnsReg prod_inst prod_result)
(ConsumesAndProducesFlags.SideEffect middle_inst)
@@ -948,7 +1076,6 @@
(_ Unit (emit consume_inst4)))
(MultiReg.Three middle_result (value_regs_get consume_result 0) (value_regs_get consume_result 1))))
-
;; ProducesFlags.ReturnsReg + ConsumesAndProducesFlags.ReturnsReg with all possible ConsumeFlags options
(rule (with_flags_chained (ProducesFlags.ProducesFlagsReturnsReg prod_inst prod_result)
(ConsumesAndProducesFlags.ReturnsReg middle_inst middle_result)
diff --git a/cranelift/filetests/filetests/isa/aarch64/uadd_overflow_brif.clif b/cranelift/filetests/filetests/isa/aarch64/uadd_overflow_brif.clif
new file mode 100644
index 0000000000..e08351022c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/uadd_overflow_brif.clif
@@ -0,0 +1,245 @@
+test compile precise-output
+set opt_level=speed
+target aarch64
+
+function %uadd_bool_unused(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2, v3 = uadd_overflow v0, v1
+ brif v3, block1, block2
+
+block1:
+ v4 = iconst.i64 1
+ return v4
+
+block2:
+ v5 = iconst.i64 2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds x5, x0, x1
+; b.hs label2 ; b label1
+; block1:
+; movz x0, #2
+; ret
+; block2:
+; movz x0, #1
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds x5, x0, x1
+; b.hs #0x10
+; block1: ; offset 0x8
+; mov x0, #2
+; ret
+; block2: ; offset 0x10
+; mov x0, #1
+; ret
+
+function %uadd_bool_unused_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2, v3 = uadd_overflow v0, v1
+ brif v3, block1, block2
+
+block1:
+ v4 = iconst.i32 1
+ return v4
+
+block2:
+ v5 = iconst.i32 2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds w5, w0, w1
+; b.hs label2 ; b label1
+; block1:
+; movz w0, #2
+; ret
+; block2:
+; movz w0, #1
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds w5, w0, w1
+; b.hs #0x10
+; block1: ; offset 0x8
+; mov w0, #2
+; ret
+; block2: ; offset 0x10
+; mov w0, #1
+; ret
+
+function %uadd_used_in_dominated_block(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2, v3 = uadd_overflow v0, v1
+ brif v3, block1, block2
+
+block1:
+ v4 = iadd v2, v0
+ return v4
+
+block2:
+ v5 = iconst.i32 2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds w5, w0, w1
+; b.hs label2 ; b label1
+; block1:
+; movz w0, #2
+; ret
+; block2:
+; add w0, w5, w0
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds w5, w0, w1
+; b.hs #0x10
+; block1: ; offset 0x8
+; mov w0, #2
+; ret
+; block2: ; offset 0x10
+; add w0, w5, w0
+; ret
+
+function %uadd_used_between_def_and_branch(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2, v3 = uadd_overflow v0, v1
+ store.i64 v2, v1
+ brif v3, block1, block2
+
+block1:
+ return v2
+
+block2:
+ v6 = iconst.i64 2
+ return v6
+}
+
+; VCode:
+; block0:
+; add x5, x0, x1
+; mov x7, x5
+; str x7, [x1]
+; adds x6, x0, x1
+; b.hs label2 ; b label1
+; block1:
+; movz x0, #2
+; ret
+; block2:
+; mov x0, x7
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; add x5, x0, x1
+; mov x7, x5
+; str x7, [x1] ; trap: heap_oob
+; adds x6, x0, x1
+; b.hs #0x1c
+; block1: ; offset 0x14
+; mov x0, #2
+; ret
+; block2: ; offset 0x1c
+; mov x0, x7
+; ret
+
+function %uflags_materialized(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ v5 = uextend.i64 v4
+ v6 = iadd v5, v2
+ brif v4, block1, block2
+
+block1:
+ v7 = iadd v3, v6
+ return v7
+
+block2:
+ v8 = iconst.i64 2
+ return v8
+}
+
+; VCode:
+; block0:
+; adds x8, x0, x1
+; cset x10, hs
+; adds x9, x0, x1
+; b.hs label2 ; b label1
+; block1:
+; movz x0, #2
+; ret
+; block2:
+; add x11, x2, x10, UXTB
+; add x0, x8, x11
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds x8, x0, x1
+; cset x10, hs
+; adds x9, x0, x1
+; b.hs #0x18
+; block1: ; offset 0x10
+; mov x0, #2
+; ret
+; block2: ; offset 0x18
+; add x11, x2, w10, uxtb
+; add x0, x8, x11
+; ret
+
+function %ubranch_in_different_block(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2, v3 = uadd_overflow v0, v1
+ v4 = iadd v2, v0
+ jump block2
+
+block1:
+ return v4
+
+block2:
+ brif v3, block1, block3
+
+block3:
+ return v2
+}
+
+; VCode:
+; block0:
+; add w5, w0, w1
+; mov x7, x5
+; b label1
+; block1:
+; adds w6, w0, w1
+; b.hs label3 ; b label2
+; block2:
+; mov x0, x7
+; ret
+; block3:
+; mov x13, x7
+; add w0, w13, w0
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; add w5, w0, w1
+; mov x7, x5
+; block1: ; offset 0x8
+; adds w6, w0, w1
+; b.hs #0x18
+; block2: ; offset 0x10
+; mov x0, x7
+; ret
+; block3: ; offset 0x18
+; mov x13, x7
+; add w0, w13, w0
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/aarch64/uadd_overflow_flag_consumers.clif b/cranelift/filetests/filetests/isa/aarch64/uadd_overflow_flag_consumers.clif
new file mode 100644
index 0000000000..298b983b34
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/uadd_overflow_flag_consumers.clif
@@ -0,0 +1,111 @@
+test compile precise-output
+set opt_level=speed
+target aarch64
+
+function %uadd_flags_trapz(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ trapz v4, user1
+ v5 = iadd v3, v2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds x5, x0, x1
+; b.lo #trap=user1
+; add x0, x5, x2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds x5, x0, x1
+; b.lo #0x10
+; add x0, x5, x2
+; ret
+; udf #0xc11f ; trap: user1
+
+function %uadd_flags_trapz_i32(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+ v3, v4 = uadd_overflow v0, v1
+ trapz v4, user1
+ v5 = iadd v3, v2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds w5, w0, w1
+; b.lo #trap=user1
+; add w0, w5, w2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds w5, w0, w1
+; b.lo #0x10
+; add w0, w5, w2
+; ret
+; udf #0xc11f ; trap: user1
+
+function %uadd_flags_trapnz(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ trapnz v4, user1
+ v5 = iadd v3, v2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds x5, x0, x1
+; b.hs #trap=user1
+; add x0, x5, x2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds x5, x0, x1
+; b.hs #0x10
+; add x0, x5, x2
+; ret
+; udf #0xc11f ; trap: user1
+
+function %uadd_flags_select(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ v5 = select v4, v3, v2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds x4, x0, x1
+; csel x0, x4, x2, hs
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds x4, x0, x1
+; csel x0, x4, x2, hs
+; ret
+
+function %uadd_flags_select_spectre_guard(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ v5 = select_spectre_guard v4, v3, v2
+ return v5
+}
+
+; VCode:
+; block0:
+; adds x4, x0, x1
+; csel x0, x4, x2, hs
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; adds x4, x0, x1
+; csel x0, x4, x2, hs
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/x64/uadd_overflow_brif.clif b/cranelift/filetests/filetests/isa/x64/uadd_overflow_brif.clif
new file mode 100644
index 0000000000..491213d2d1
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/uadd_overflow_brif.clif
@@ -0,0 +1,324 @@
+test compile precise-output
+set opt_level=speed
+target x86_64
+
+function %uadd_bool_unused(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2, v3 = uadd_overflow v0, v1
+ brif v3, block1, block2
+
+block1:
+ v4 = iconst.i64 1
+ return v4
+
+block2:
+ v5 = iconst.i64 2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addq %rsi, %rdi
+; jb label2; j label1
+; block1:
+; movl $0x2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; movl $0x1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; addq %rsi, %rdi
+; jb 0x17
+; block2: ; offset 0xd
+; movl $2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x17
+; movl $1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uadd_bool_unused_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2, v3 = uadd_overflow v0, v1
+ brif v3, block1, block2
+
+block1:
+ v4 = iconst.i32 1
+ return v4
+
+block2:
+ v5 = iconst.i32 2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addl %esi, %edi
+; jb label2; j label1
+; block1:
+; movl $0x2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; movl $0x1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; addl %esi, %edi
+; jb 0x16
+; block2: ; offset 0xc
+; movl $2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x16
+; movl $1, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uadd_used_in_dominated_block(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2, v3 = uadd_overflow v0, v1
+ brif v3, block1, block2
+
+block1:
+ v4 = iadd v2, v0
+ return v4
+
+block2:
+ v5 = iconst.i32 2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq %rdi, %r8
+; addl %esi, %r8d
+; jb label2; j label1
+; block1:
+; movl $0x2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; leal (%r8, %rdi), %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq %rdi, %r8
+; addl %esi, %r8d
+; jb 0x1a
+; block2: ; offset 0x10
+; movl $2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x1a
+; leal (%r8, %rdi), %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uadd_used_between_def_and_branch(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2, v3 = uadd_overflow v0, v1
+ store.i64 v2, v1
+ brif v3, block1, block2
+
+block1:
+ return v2
+
+block2:
+ v6 = iconst.i64 2
+ return v6
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq %rdi, %rax
+; addq %rsi, %rax
+; movq %rax, (%rsi)
+; addq %rsi, %rdi
+; jb label2; j label1
+; block1:
+; movl $0x2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq %rdi, %rax
+; addq %rsi, %rax
+; movq %rax, (%rsi) ; trap: heap_oob
+; addq %rsi, %rdi
+; jb 0x20
+; block2: ; offset 0x16
+; movl $2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x20
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uflags_materialized(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ v5 = uextend.i64 v4
+ v6 = iadd v5, v2
+ brif v4, block1, block2
+
+block1:
+ v7 = iadd v3, v6
+ return v7
+
+block2:
+ v8 = iconst.i64 2
+ return v8
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq %rdi, %r9
+; addq %rsi, %r9
+; setb %r11b
+; addq %rsi, %rdi
+; jb label2; j label1
+; block1:
+; movl $0x2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; movzbq %r11b, %rax
+; leaq (%rax, %rdx), %rax
+; leaq (%r9, %rax), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq %rdi, %r9
+; addq %rsi, %r9
+; setb %r11b
+; addq %rsi, %rdi
+; jb 0x21
+; block2: ; offset 0x17
+; movl $2, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x21
+; movzbq %r11b, %rax
+; addq %rdx, %rax
+; addq %r9, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %ubranch_in_different_block(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+ v2, v3 = uadd_overflow v0, v1
+ v4 = iadd v2, v0
+ jump block2
+
+block1:
+ return v4
+
+block2:
+ brif v3, block1, block3
+
+block3:
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq %rdi, %rax
+; addl %esi, %eax
+; jmp label1
+; block1:
+; movq %rdi, %r8
+; addl %esi, %r8d
+; jb label3; j label2
+; block2:
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3:
+; leal (%rax, %rdi), %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq %rdi, %rax
+; addl %esi, %eax
+; block2: ; offset 0x9
+; movq %rdi, %r8
+; addl %esi, %r8d
+; jb 0x1a
+; block3: ; offset 0x15
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block4: ; offset 0x1a
+; addl %edi, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
diff --git a/cranelift/filetests/filetests/isa/x64/uadd_overflow_flag_consumers.clif b/cranelift/filetests/filetests/isa/x64/uadd_overflow_flag_consumers.clif
new file mode 100644
index 0000000000..c9956d30b8
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/uadd_overflow_flag_consumers.clif
@@ -0,0 +1,160 @@
+test compile precise-output
+set opt_level=speed
+target x86_64
+
+function %uadd_flags_trapz(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ trapz v4, user1
+ v5 = iadd v3, v2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addq %rsi, %rdi
+; jnb #trap=user1
+; leaq (%rdi, %rdx), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; addq %rsi, %rdi
+; jae 0x16
+; leaq (%rdi, %rdx), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; ud2 ; trap: user1
+
+function %uadd_flags_trapz_i32(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+ v3, v4 = uadd_overflow v0, v1
+ trapz v4, user1
+ v5 = iadd v3, v2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addl %esi, %edi
+; jnb #trap=user1
+; leal (%rdi, %rdx), %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; addl %esi, %edi
+; jae 0x14
+; leal (%rdi, %rdx), %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; ud2 ; trap: user1
+
+function %uadd_flags_trapnz(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ trapnz v4, user1
+ v5 = iadd v3, v2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addq %rsi, %rdi
+; jb #trap=user1
+; leaq (%rdi, %rdx), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; addq %rsi, %rdi
+; jb 0x16
+; leaq (%rdi, %rdx), %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; ud2 ; trap: user1
+
+function %uadd_flags_select(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ v5 = select v4, v3, v2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addq %rsi, %rdi
+; movq %rdx, %rax
+; cmovbq %rdi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; addq %rsi, %rdi
+; movq %rdx, %rax
+; cmovbq %rdi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %uadd_flags_select_spectre_guard(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+ v3, v4 = uadd_overflow v0, v1
+ v5 = select_spectre_guard v4, v3, v2
+ return v5
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addq %rsi, %rdi
+; movq %rdx, %rax
+; cmovbq %rdi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; addq %rsi, %rdi
+; movq %rdx, %rax
+; cmovbq %rdi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.wat
index 251ca92806..ace7585acf 100644
--- a/tests/disas/debug-exceptions.wat
+++ b/tests/disas/debug-exceptions.wat
@@ -31,35 +31,35 @@
;; stur x0, [sp, #0x18]
;; mov x0, sp
;; cmp x0, x1
-;; b.lo #0x218
+;; b.lo #0x210
;; 48: stur x2, [sp]
;; mov x0, x2
;; stur x2, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x35, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [67, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [65, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [65, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [63, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3d, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [63, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [61, 1, 0, 148]
;; mov w1, #0x2a
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3f, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [60, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [58, 1, 0, 148]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x40, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [59, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [57, 1, 0, 148]
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [57, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [55, 1, 0, 148]
;; ldur x2, [sp, #0x10]
-;; bl #0x4c8
+;; bl #0x4c0
;; 88: ldur x0, [sp, #0x10]
;; mov x19, x2
;; ldr x0, [x0, #0x20]
@@ -68,43 +68,41 @@
;; add x1, x1, #0x20
;; ldr w2, [x0, #4]
;; cmp x1, x2
-;; b.hi #0x198
+;; b.hi #0x190
;; ac: ldur x4, [sp, #0x18]
-;; add w2, w3, #0x20
-;; str w2, [x0]
-;; mov w5, #2
-;; movk w5, #0x400, lsl #16
-;; ldr x6, [x4, #0x20]
-;; add x1, x6, w3, uxtw
-;; str w5, [x6, w3, uxtw]
+;; add w1, w3, #0x20
+;; str w1, [x0]
+;; mov w2, #2
+;; movk w2, #0x400, lsl #16
+;; ldr x5, [x4, #0x20]
+;; add x1, x5, w3, uxtw
+;; str w2, [x5, w3, uxtw]
;; ldur x0, [sp, #0x10]
;; ldr x4, [x0, #0x28]
;; ldr w4, [x4, #8]
;; str w4, [x1, #4]
-;; mov x5, #0x20
-;; str w5, [x1, #8]
-;; mov w7, #0x2a
-;; str w7, [x1, #0x18]
+;; mov x4, #0x20
+;; str w4, [x1, #8]
+;; mov w6, #0x2a
+;; str w6, [x1, #0x18]
;; mov x2, x19
;; str w2, [x1, #0x10]
-;; mov w9, #0
-;; str w9, [x1, #0x14]
+;; mov w8, #0
+;; str w8, [x1, #0x14]
;; ldur x2, [sp, #0x10]
-;; bl #0x500
+;; bl #0x4f8
;; ├─╼ exception frame offset: SP = FP - 0xb0
;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x108
-;; b #0x1d0
-;; 108: mov w14, w0
-;; mov x15, #0x20
-;; adds x13, x14, x15
-;; cset x15, hs
-;; tst w15, #0xff
-;; b.ne #0x200
-;; 120: ldur x2, [sp, #0x18]
-;; ldr x1, [x2, #0x28]
-;; cmp x13, x1
-;; b.hi #0x1e8
-;; 130: ldr x1, [x2, #0x20]
+;; b #0x1c8
+;; 108: mov w13, w0
+;; mov x14, #0x20
+;; adds x13, x13, x14
+;; b.hs #0x1f8
+;; 118: ldur x2, [sp, #0x18]
+;; ldr x14, [x2, #0x28]
+;; cmp x13, x14
+;; b.hi #0x1e0
+;; 128: ldr x1, [x2, #0x20]
;; add x0, x1, w0, uxtw
;; ldr w0, [x0, #0x18]
;; stur w0, [sp, #8]
@@ -112,13 +110,13 @@
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x48, slot at FP-0xb0, locals , stack I32 @ slot+0x8
;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [7, 1, 0, 148]
-;; ldur x1, [sp, #0x10]
-;; ldr x0, [x1, #0x38]
-;; ldr x2, [x1, #0x48]
+;; ldur x14, [sp, #0x10]
+;; ldr x0, [x14, #0x38]
+;; ldr x2, [x14, #0x48]
;; ldur x3, [sp, #0x10]
;; blr x0
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; 15c: ldur x0, [sp, #0x10]
+;; 154: ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8
;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [0, 1, 0, 148]
@@ -137,45 +135,45 @@
;; ldp x27, x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 198: mov w3, #2
-;; 19c: movk w3, #0x400, lsl #16
-;; 1a0: ldur x0, [sp, #0x10]
-;; 1a4: ldr x1, [x0, #0x28]
-;; 1a8: ldr w4, [x1, #8]
-;; 1ac: mov w5, #0x20
-;; 1b0: mov w6, #0x10
-;; 1b4: ldur x2, [sp, #0x10]
-;; 1b8: bl #0x3f4
-;; 1bc: ldur x4, [sp, #0x18]
-;; 1c0: ldr x1, [x4, #0x20]
-;; 1c4: add x1, x1, w2, uxtw
-;; 1c8: mov x3, x2
-;; 1cc: b #0xe4
-;; 1d0: mov w3, #9
+;; 190: mov w3, #2
+;; 194: movk w3, #0x400, lsl #16
+;; 198: ldur x0, [sp, #0x10]
+;; 19c: ldr x0, [x0, #0x28]
+;; 1a0: ldr w4, [x0, #8]
+;; 1a4: mov w5, #0x20
+;; 1a8: mov w6, #0x10
+;; 1ac: ldur x2, [sp, #0x10]
+;; 1b0: bl #0x3ec
+;; 1b4: ldur x4, [sp, #0x18]
+;; 1b8: ldr x0, [x4, #0x20]
+;; 1bc: add x1, x0, w2, uxtw
+;; 1c0: mov x3, x2
+;; 1c4: b #0xe4
+;; 1c8: mov w3, #9
+;; 1cc: ldur x2, [sp, #0x10]
+;; 1d0: bl #0x454
;; 1d4: ldur x2, [sp, #0x10]
-;; 1d8: bl #0x45c
-;; 1dc: ldur x2, [sp, #0x10]
-;; 1e0: bl #0x494
+;; 1d8: bl #0x48c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; 1e4: udf #0xc11f
-;; 1e8: mov w3, #0xfe
+;; 1dc: udf #0xc11f
+;; 1e0: mov w3, #0xfe
+;; 1e4: ldur x2, [sp, #0x10]
+;; 1e8: bl #0x454
;; 1ec: ldur x2, [sp, #0x10]
-;; 1f0: bl #0x45c
-;; 1f4: ldur x2, [sp, #0x10]
-;; 1f8: bl #0x494
+;; 1f0: bl #0x48c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; 1fc: udf #0xc11f
-;; 200: mov w3, #0xfe
+;; 1f4: udf #0xc11f
+;; 1f8: mov w3, #0xfe
+;; 1fc: ldur x2, [sp, #0x10]
+;; 200: bl #0x454
;; 204: ldur x2, [sp, #0x10]
-;; 208: bl #0x45c
-;; 20c: ldur x2, [sp, #0x10]
-;; 210: bl #0x494
+;; 208: bl #0x48c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; 214: udf #0xc11f
-;; 218: stur x2, [sp, #0x10]
-;; 21c: mov w3, #0
-;; 220: bl #0x45c
-;; 224: ldur x2, [sp, #0x10]
-;; 228: bl #0x494
+;; 20c: udf #0xc11f
+;; 210: stur x2, [sp, #0x10]
+;; 214: mov w3, #0
+;; 218: bl #0x454
+;; 21c: ldur x2, [sp, #0x10]
+;; 220: bl #0x48c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x34, slot at FP-0xb0, locals , stack
-;; 22c: udf #0xc11f
+;; 224: udf #0xc11f