A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: Fold branch-to-trap optimization into the egraphs pass (#13688)
Nick Fitzgeralde04ccb5be0bb
diff --git a/cranelift/codegen/src/branch_to_trap.rs b/cranelift/codegen/src/branch_to_trap.rsnew file mode 100644index 0000000000..4e33a2f36e--- /dev/null+++ b/cranelift/codegen/src/branch_to_trap.rs@@ -0,0 +1,87 @@+//! Analysis for rewriting branch-to-unconditional-trap into conditional trap+//! instructions.+//!+//! Given this instruction:+//!+//! ```clif+//! brif v0, block1, block2+//! ```+//!+//! If we know that `block1` does nothing but immediately trap then we can+//! rewrite that `brif` into the following:+//!+//! ```clif+//! trapnz v0, <trapcode>+//! jump block2+//! ```+//!+//! (And we can do the equivalent with `trapz` if `block2` immediately traps).+//!+//! This transformation allows for the conditional trap instructions to be GVN'd+//! and for our egraphs mid-end to generally better optimize the program. We+//! additionally have better codegen in our backends for `trapz`/`trapnz` than+//! branches to unconditional traps.+//!+//! This module only provides the *analysis* of which blocks are "just trap"+//! blocks; the actual rewrite is performed by `simplify_skeleton` ISLE rules in+//! the egraph pass, which consult this analysis via the `just_trap_block` ISLE+//! constructor.++use crate::FxHashMap;+use crate::inst_predicates::is_pure_for_egraph;+use crate::ir::{self, InstructionData, Opcode};+use cranelift_entity::EntitySet;++/// On-demand, memoized analysis of which blocks are "just trap" blocks.+///+/// A block is a "just trap" block when its terminator is an unconditional+/// `trap` and all of its other instructions in the block are pure.+///+/// Results are memoized so that it does not matter in which order blocks are+/// analyzed or instructions are processed; callers can ask for the analysis+/// result of any block at any time.+#[derive(Default)]+pub struct BranchToTrapAnalysis {+ /// The set of blocks we have already analyzed.+ analyzed_blocks: EntitySet<ir::Block>,++ /// Given that we have already analyzed a block and found it to be a+ /// just-trap block, what is its trap code?+ just_trap_block_codes: FxHashMap<ir::Block, ir::TrapCode>,+}++impl BranchToTrapAnalysis {+ /// Determine whether `block` is a "just trap" block and, if so, return the+ /// trap code of its terminating `trap` instruction.+ pub fn analyze_block(&mut self, func: &ir::Function, block: ir::Block) -> Option<ir::TrapCode> {+ if self.analyzed_blocks.insert(block) {+ let code = Self::analyze_block_impl(func, block);+ if let Some(code) = code {+ let old_entry = self.just_trap_block_codes.insert(block, code);+ debug_assert!(old_entry.is_none());+ }+ code+ } else {+ self.just_trap_block_codes.get(&block).copied()+ }+ }++ fn analyze_block_impl(func: &ir::Function, block: ir::Block) -> Option<ir::TrapCode> {+ let last = func.layout.last_inst(block)?;+ let code = match func.dfg.insts[last] {+ InstructionData::Trap {+ opcode: Opcode::Trap,+ code,+ } => code,+ _ => return None,+ };++ for inst in func.layout.block_insts(block) {+ if inst != last && !is_pure_for_egraph(func, inst) {+ return None;+ }+ }++ Some(code)+ }+}diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rsindex 177563b8d3..a81291bd57 100644--- a/cranelift/codegen/src/context.rs+++ b/cranelift/codegen/src/context.rs@@ -16,7 +16,6 @@ use crate::flowgraph::ControlFlowGraph;use crate::inline::{Inline, do_inlining};use crate::ir::Function;use crate::isa::TargetIsa;-use crate::legalizer::simple_legalize;use crate::loop_analysis::LoopAnalysis;use crate::machinst::{CompiledCode, CompiledCodeStencil};use crate::nan_canonicalization::do_nan_canonicalization;@@ -173,7 +172,7 @@ impl Context {self.canonicalize_nans(isa)?;}- self.legalize(isa)?;+ self.verify_if(isa)?;self.compute_cfg();self.compute_domtree();@@ -293,19 +292,6 @@ impl Context {self.verify_if(isa)}- /// Run the legalizer for `isa` on the function.- pub fn legalize(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {- // Legalization invalidates the domtree and loop_analysis by mutating the CFG.- // TODO: Avoid doing this when legalization doesn't actually mutate the CFG.- self.domtree.clear();- self.loop_analysis.clear();- self.cfg.clear();-- // Run some specific legalizations only.- simple_legalize(&mut self.func);- self.verify_if(isa)- }-/// Compute the control flow graph.pub fn compute_cfg(&mut self) {self.cfg.compute(&self.func)diff --git a/cranelift/codegen/src/egraph/mod.rs b/cranelift/codegen/src/egraph/mod.rsindex cb083d58ce..6a4525cb7c 100644--- a/cranelift/codegen/src/egraph/mod.rs+++ b/cranelift/codegen/src/egraph/mod.rs@@ -2,6 +2,7 @@use crate::FxHashSet;use crate::alias_analysis::{AliasAnalysis, LastStores, OptResult};+use crate::branch_to_trap::BranchToTrapAnalysis;use crate::ctxhash::{CtxEq, CtxHash, NullCtx};use crate::cursor::{Cursor, CursorPosition, FuncCursor};use crate::dominator_tree::DominatorTree;@@ -112,6 +113,10 @@ pub struct EgraphPass<'a> {/// The control flow graph, used when eliminating unreachable code/// after branch simplification.cfg: &'a mut ControlFlowGraph,+ /// Branch-to-trap analysis: determines which blocks are "just trap" blocks,+ /// used by `simplify_skeleton` rules to rewrite conditional branches to+ /// trapping blocks into conditional traps.+ branch_to_trap_analysis: BranchToTrapAnalysis,/// Which Values do we want to rematerialize in each block where/// they're used?remat_values: FxHashSet<Value>,@@ -148,6 +153,7 @@ wheredomtree: &'opt DominatorTree,pub(crate) alias_analysis: &'opt mut AliasAnalysis<'analysis>,pub(crate) alias_analysis_state: &'opt mut LastStores,+ pub(crate) branch_to_trap_analysis: &'opt mut BranchToTrapAnalysis,ctrl_plane: &'opt mut ControlPlane,// Held locally during optimization of one node (recursively):pub(crate) rewrite_depth: usize,@@ -642,6 +648,28 @@ wherereturn Some(simplification);}+ // `ReplaceBranchCond` is unconditionally accepted: the opcode+ // and successors don't change, so we can't use the cost-based+ // ranking the other variants do (skeleton cost can't consider+ // operand costs).+ SkeletonInstSimplification::ReplaceBranchCond { cond } => {+ log::trace!(" -> simplify_skeleton: replace condition operand with {cond}");+ return Some(SkeletonInstSimplification::ReplaceBranchCond { cond });+ }+ // `ReplaceWithTwo` (e.g. rewriting a branch-to-trap-block into+ // a conditional trap + jump) is also unconditionally accepted:+ // these rules are always an improvement, but due to its shape,+ // cannot participate in the cost-based ranking other variants+ // use.+ SkeletonInstSimplification::ReplaceWithTwo { first, second } => {+ log::trace!(+ " -> simplify_skeleton: replace inst with `{}; {}`",+ ctx.func.dfg.display_inst(first),+ ctx.func.dfg.display_inst(second),+ );+ return Some(SkeletonInstSimplification::ReplaceWithTwo { first, second });+ }+// For instruction replacement simplification, we want to check// that the replacements define the same number and types of// values as the original instruction, and also determine@@ -661,16 +689,6 @@ where);(inst, Some(val))}- // `ReplaceBranchCond` is unconditionally accepted — the- // opcode and successors don't change, so we can't use the- // cost-based ranking the other variants do (replacing the- // condition with a cheaper sub-expression keeps the same- // opcode/arity, hence the same skeleton cost). The first such- // candidate wins; ISLE rule ordering picks the form.- SkeletonInstSimplification::ReplaceBranchCond { cond } => {- log::trace!(" -> simplify_skeleton: replace condition operand with {cond}");- return Some(SkeletonInstSimplification::ReplaceBranchCond { cond });- }};if cfg!(debug_assertions) {@@ -730,6 +748,7 @@ impl<'a> EgraphPass<'a> {alias_analysis,ctrl_plane,cfg,+ branch_to_trap_analysis: BranchToTrapAnalysis::default(),stats: Stats::default(),remat_values: FxHashSet::default(),}@@ -920,6 +939,7 @@ impl<'a> EgraphPass<'a> {domtree: &self.domtree,alias_analysis: self.alias_analysis,alias_analysis_state: &mut alias_analysis_state,+ branch_to_trap_analysis: &mut self.branch_to_trap_analysis,ctrl_plane: self.ctrl_plane,optimized_values: Default::default(),optimized_insts: Default::default(),@@ -977,11 +997,50 @@ impl<'a> EgraphPass<'a> {value_to_opt_value: &mut SecondaryMap<Value, Value>,old_inst: Inst,) {- let mut forward_val = |cursor: &mut FuncCursor, old_val, new_val| {+ // Redirect uses of `old_val` to `new_val`.+ let forward_val = |cursor: &mut FuncCursor<'_>,+ value_to_opt_value: &mut SecondaryMap<_, _>,+ old_val,+ new_val| {cursor.func.dfg.change_to_alias(old_val, new_val);value_to_opt_value[old_val] = new_val;};+ // Values created during skeleton-instruction simplification are created+ // after we've already computed the `value_to_opt_value` map and are+ // therefore missing entries within it. These values are already+ // optimized, so map them to themselves.+ let self_map_operands =+ |dfg: &DataFlowGraph, value_to_opt_value: &mut SecondaryMap<_, _>, inst| {+ for val in dfg.inst_values(inst) {+ debug_assert!(+ value_to_opt_value[val] == val+ || value_to_opt_value[val] == Value::reserved_value()+ );+ value_to_opt_value[val] = core::cmp::min(value_to_opt_value[val], val);+ }+ };++ // Any new instructions produced by a simplification inherit the source+ // location of the instruction they replace.+ let old_srcloc = cursor.func.srclocs[old_inst];+ // NB: But we only inherit the source location when it is non-default to+ // avoid extending the `SecondaryMap`.+ let old_srcloc = if old_srcloc.is_default() {+ None+ } else {+ Some(old_srcloc)+ };++ // Rewind the cursor to just before `inst` so that the main loop+ // re-processes it on its next iteration. This lets a freshly+ // produced/modified skeleton instruction be GVN'd and/or simplified+ // further.+ fn reprocess_from(cursor: &mut FuncCursor, inst: Inst) {+ cursor.goto_inst(inst);+ cursor.prev_inst();+ }+let (new_inst, new_val) = match simplification {SkeletonInstSimplification::Remove => {cursor.remove_inst_and_step_back();@@ -991,11 +1050,9 @@ impl<'a> EgraphPass<'a> {cursor.remove_inst_and_step_back();let old_val = cursor.func.dfg.first_result(old_inst);cursor.func.dfg.detach_inst_results(old_inst);- forward_val(cursor, old_val, val);+ forward_val(cursor, value_to_opt_value, old_val, val);return;}- SkeletonInstSimplification::Replace { inst } => (inst, None),- SkeletonInstSimplification::ReplaceWithVal { inst, val } => (inst, Some(val)),SkeletonInstSimplification::ReplaceBranchCond { cond } => {// Swap the condition operand (argument 0) of the existing// conditional skeleton instruction in place. The opcode and any@@ -1007,10 +1064,51 @@ impl<'a> EgraphPass<'a> {crate::ir::Opcode::Brif | crate::ir::Opcode::Trapz | crate::ir::Opcode::Trapnz,));cursor.func.dfg.inst_args_mut(old_inst)[0] = cond;+ // Re-process the modified instruction so that, e.g., another+ // truthiness-preserving layer can be stripped from its+ // condition or it can be GVN'd.+ self_map_operands(&cursor.func.dfg, value_to_opt_value, old_inst);+ reprocess_from(cursor, old_inst);return;}+ SkeletonInstSimplification::ReplaceWithTwo { first, second } => {+ // We don't forward result values for `ReplaceWithTwo` -- and it+ // isn't clear what that would mean when both the new+ // instructions have result values -- so we only support+ // instructions without results here.+ debug_assert!(cursor.func.dfg.inst_results(old_inst).is_empty());+ debug_assert!(cursor.func.dfg.inst_results(first).is_empty());+ debug_assert!(cursor.func.dfg.inst_results(second).is_empty());++ // If the instruction we're replacing is a block terminator,+ // then the trailing new instruction must also be a terminator,+ // so the block remains well-formed.+ debug_assert!(+ !cursor.func.dfg.insts[old_inst].opcode().is_terminator()+ || cursor.func.dfg.insts[second].opcode().is_terminator()+ );++ if let Some(old_srcloc) = old_srcloc {+ cursor.func.srclocs[first] = old_srcloc;+ cursor.func.srclocs[second] = old_srcloc;+ }++ cursor.insert_inst(first);+ cursor.replace_inst(second);+ self_map_operands(&cursor.func.dfg, value_to_opt_value, first);+ self_map_operands(&cursor.func.dfg, value_to_opt_value, second);+ reprocess_from(cursor, first);+ return;+ }++ SkeletonInstSimplification::Replace { inst } => (inst, None),+ SkeletonInstSimplification::ReplaceWithVal { inst, val } => (inst, Some(val)),};+ if let Some(old_srcloc) = old_srcloc {+ cursor.func.srclocs[new_inst] = old_srcloc;+ }+// Replace the old instruction with the new one.cursor.replace_inst(new_inst);@@ -1029,10 +1127,11 @@ impl<'a> EgraphPass<'a> {for i in 0..cursor.func.dfg.inst_results(old_inst).len() {let old_val = cursor.func.dfg.inst_results(old_inst)[i];let new_val = next_new_val(&cursor.func.dfg);- forward_val(cursor, old_val, new_val);+ forward_val(cursor, value_to_opt_value, old_val, new_val);}- cursor.goto_inst(new_inst);+ self_map_operands(&cursor.func.dfg, value_to_opt_value, new_inst);+ reprocess_from(cursor, new_inst);}/// Scoped elaboration: compute a final ordering of op computationdiff --git a/cranelift/codegen/src/legalizer/branch_to_trap.rs b/cranelift/codegen/src/legalizer/branch_to_trap.rsdeleted file mode 100644index c35f03e864..0000000000--- a/cranelift/codegen/src/legalizer/branch_to_trap.rs+++ /dev/null@@ -1,98 +0,0 @@-//! Rewrite branch-to-unconditional-trap into conditional trap instructions.-//!-//! Given this instruction:-//!-//! ```clif-//! brif v0, block1, block2-//! ```-//!-//! If we know that `block1` does nothing but immediately trap then we can-//! rewrite that `brif` into the following:-//!-//! ```clif-//! trapz v0, <trapcode>-//! jump block2-//! ```-//!-//! (And we can do the equivalent with `trapz` if `block2` immediately traps).-//!-//! This transformation allows for the conditional trap instructions to be GVN'd-//! and for our egraphs mid-end to generally better optimize the program. We-//! additionally have better codegen in our backends for `trapz` than branches-//! to unconditional traps.--use super::*;--#[derive(Default)]-pub struct BranchToTrap {- /// The set of blocks that contain exactly one unconditional trap- /// instruction.- just_trap_blocks: EntitySet<ir::Block>,-}--impl BranchToTrap {- /// Analyze the given block.- ///- /// The `block` must be terminated by a `trap` instruction.- pub fn analyze_trapping_block(&mut self, func: &ir::Function, block: ir::Block) {- if func.layout.block_contains_exactly_one_inst(block) {- self.just_trap_blocks.insert(block);- }- }-- fn just_trap_block_code(&self, func: &ir::Function, block: ir::Block) -> ir::TrapCode {- debug_assert!(self.just_trap_blocks.contains(block));- debug_assert!(func.layout.block_contains_exactly_one_inst(block));- let inst = func.layout.first_inst(block).unwrap();- match func.dfg.insts[inst] {- InstructionData::Trap { code, .. } => code,- _ => unreachable!(),- }- }-- /// Process a `brif` instruction, potentially performing our rewrite.- ///- /// The `inst` must be a `brif` containing the given `arg` and `blocks`.- pub fn process_brif(- &self,- func: &mut ir::Function,- inst: ir::Inst,- arg: ir::Value,- blocks: [ir::BlockCall; 2],- ) {- let consequent = blocks[0].block(&func.dfg.value_lists);- let alternative = blocks[1].block(&func.dfg.value_lists);-- if self.just_trap_blocks.contains(consequent) {- let mut pos = FuncCursor::new(func);- pos.use_srcloc(- pos.func- .layout- .first_inst(consequent)- .expect("just-trap blocks have exactly one inst"),- );- pos.goto_inst(inst);-- let code = self.just_trap_block_code(pos.func, consequent);- pos.ins().trapnz(arg, code);-- let args: SmallVec<[_; 8]> = blocks[1].args(&pos.func.dfg.value_lists).collect();- pos.func.replace(inst).jump(alternative, &args);- } else if self.just_trap_blocks.contains(alternative) {- let mut pos = FuncCursor::new(func);- pos.use_srcloc(- pos.func- .layout- .first_inst(alternative)- .expect("just-trap blocks have exactly one inst"),- );- pos.goto_inst(inst);-- let code = self.just_trap_block_code(pos.func, alternative);- pos.ins().trapz(arg, code);-- let args: SmallVec<[_; 8]> = blocks[0].args(&pos.func.dfg.value_lists).collect();- pos.func.replace(inst).jump(consequent, &args);- }- }-}diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rsdeleted file mode 100644index ed09a05b74..0000000000--- a/cranelift/codegen/src/legalizer/mod.rs+++ /dev/null@@ -1,75 +0,0 @@-//! Legalize instructions.-//!-//! A legal instruction is one that can be mapped directly to a machine code instruction for the-//! target ISA. The `legalize_function()` function takes as input any function and transforms it-//! into an equivalent function using only legal instructions.-//!-//! The characteristics of legal instructions depend on the target ISA, so any given instruction-//! can be legal for one ISA and illegal for another.-//!-//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map-//! which provides a legal encoding recipe for every instruction.-//!-//! The legalizer does not deal with register allocation constraints. These constraints are derived-//! from the encoding recipes, and solved later by the register allocator.--use crate::cursor::{Cursor, FuncCursor};-use crate::ir::{self, InstBuilder, InstructionData};-use crate::trace;-use cranelift_entity::EntitySet;-use smallvec::SmallVec;--mod branch_to_trap;-use self::branch_to_trap::BranchToTrap;--/// A simple, naive backwards walk over every instruction in every block in the-/// function's layout.-///-/// This does not guarantee any kind of reverse post-order visitation or-/// anything like that, it is just iterating over blocks in reverse layout-/// order, not any kind of control-flow graph visitation order.-fn backward_walk(- func: &mut ir::Function,- mut f: impl FnMut(&mut ir::Function, ir::Block, ir::Inst),-) {- let mut pos = FuncCursor::new(func);- while let Some(block) = pos.prev_block() {- while let Some(inst) = pos.prev_inst() {- f(pos.func, block, inst);- }- }-}--/// Perform a simple legalization by expansion of the function, without-/// platform-specific transforms.-pub fn simple_legalize(func: &mut ir::Function) {- trace!("Pre-legalization function:\n{}", func.display());-- let mut branch_to_trap = BranchToTrap::default();-- // We walk the IR backwards because in practice, given the way that- // frontends tend to produce CLIF, this means we will visit in roughly- // reverse post order, which is helpful for getting the most optimizations- // out of the `branch-to-trap` pass that we can (it must analyze trapping- // blocks before it can rewrite branches to them) but the order does not- // actually affect correctness.- backward_walk(func, |func, block, inst| match func.dfg.insts[inst] {- InstructionData::Trap {- opcode: ir::Opcode::Trap,- code: _,- } => {- branch_to_trap.analyze_trapping_block(func, block);- }- InstructionData::Brif {- opcode: ir::Opcode::Brif,- arg,- blocks,- } => {- branch_to_trap.process_brif(func, inst, arg, blocks);- }-- _ => {}- });-- trace!("Post-legalization function:\n{}", func.display());-}diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rsindex 08de586ac9..0fad51d52e 100644--- a/cranelift/codegen/src/lib.rs+++ b/cranelift/codegen/src/lib.rs@@ -81,13 +81,13 @@ pub use crate::machinst::{};mod alias_analysis;+mod branch_to_trap;mod constant_hash;mod context;mod ctxhash;mod egraph;mod inst_predicates;mod isle_prelude;-mod legalizer;mod nan_canonicalization;mod opts;mod ranges;diff --git a/cranelift/codegen/src/opts.rs b/cranelift/codegen/src/opts.rsindex 3471acb2f1..ac68fb2bb3 100644--- a/cranelift/codegen/src/opts.rs+++ b/cranelift/codegen/src/opts.rs@@ -9,7 +9,7 @@ pub use crate::ir::immediates::{Ieee16, Ieee32, Ieee64, Ieee128, Imm64, Offset32use crate::ir::instructions::InstructionFormat;pub use crate::ir::types::*;pub use crate::ir::{- AtomicRmwOp, BlockCall, Constant, DynamicStackSlot, FuncRef, GlobalValue, Immediate,+ AtomicRmwOp, Block, BlockCall, Constant, DynamicStackSlot, FuncRef, GlobalValue, Immediate,InstructionData, JumpTable, MemFlagsData, Opcode, StackSlot, TrapCode, Type, Value,};use crate::isle_common_prelude_methods;@@ -263,6 +263,16 @@ impl<'a, 'b, 'c> generated_code::Context for IsleContext<'a, 'b, 'c> {}}+ fn block_call_block(&mut self, block_call: BlockCall) -> Block {+ block_call.block(&self.ctx.func.dfg.value_lists)+ }++ fn just_trap_block(&mut self, block: &Block) -> Option<TrapCode> {+ self.ctx+ .branch_to_trap_analysis+ .analyze_block(self.ctx.func, *block)+ }+fn iconst_sextend_etor(&mut self,(ty, inst_data): (Type, InstructionData),diff --git a/cranelift/codegen/src/opts/skeleton.isle b/cranelift/codegen/src/opts/skeleton.isleindex e779ce3026..b1cb5304c8 100644--- a/cranelift/codegen/src/opts/skeleton.isle+++ b/cranelift/codegen/src/opts/skeleton.isle@@ -26,6 +26,19 @@(rule (simplify_skeleton (brif (iconst_u _ 0) _ else_dest))(jump else_dest))+;; Conditional branch to a block that does nothing but trap: rewrite into a+;; conditional trap plus a jump to the surviving successor. The trapping+;; successor's block-call arguments are dropped (it just traps); the surviving+;; successor's block call (with its arguments) is reused in the new `jump`.+(rule (simplify_skeleton (brif condition consequent alternative))+ (if-let trap_code (just_trap_block (block_call_block consequent)))+ (replace_with_two (trapnz condition trap_code)+ (jump alternative)))+(rule (simplify_skeleton (brif condition consequent alternative))+ (if-let trap_code (just_trap_block (block_call_block alternative)))+ (replace_with_two (trapz condition trap_code)+ (jump consequent)))+;; Branch table with known index: resolve the destination from the;; jump table data and replace with an unconditional jump.(rule (simplify_skeleton (br_table (iconst_u _ idx) jt))diff --git a/cranelift/codegen/src/prelude_opt.isle b/cranelift/codegen/src/prelude_opt.isleindex f1c80b6e66..f333d51bd0 100644--- a/cranelift/codegen/src/prelude_opt.isle+++ b/cranelift/codegen/src/prelude_opt.isle@@ -42,6 +42,16 @@(decl pure resolve_jump_table_entry (JumpTable u64) BlockCall)(extern constructor resolve_jump_table_entry resolve_jump_table_entry)+;; Get the destination block of a block call.+(decl pure block_call_block (BlockCall) Block)+(extern constructor block_call_block block_call_block)++;; If `block` is a "just trap" block (its terminator is an unconditional `trap`+;; and all of its other instructions are pure) return the trap code of that+;; terminating `trap`.+(decl pure partial just_trap_block (Block) TrapCode)+(extern constructor just_trap_block just_trap_block)+(rule (eq ty x y) (icmp ty (IntCC.Equal) x y))(rule (ne ty x y) (icmp ty (IntCC.NotEqual) x y))(rule (ult ty x y) (icmp ty (IntCC.UnsignedLessThan) x y))@@ -103,7 +113,15 @@;;;; Use this to rewrite conditions which simplify within a boolean-test;; context (e.g. `(brif (ctz x))` can be simplified to `(brif x)`).- (ReplaceBranchCond (cond Value))))+ (ReplaceBranchCond (cond Value))++ ;; Replace the instruction being simplified with two instructions,+ ;; spliced into the layout in order: `first` then `second`.+ ;;+ ;; Neither instruction may define any results, and if the instruction+ ;; being replaced is a block terminator then `second` must also be a+ ;; block terminator.+ (ReplaceWithTwo (first Inst) (second Inst))))(decl pure inst_to_skeleton_inst_simplification (Inst) SkeletonInstSimplification)(rule (inst_to_skeleton_inst_simplification inst)@@ -123,6 +141,10 @@(rule (replace_branch_cond new_cond)(SkeletonInstSimplification.ReplaceBranchCond new_cond))+(decl pure replace_with_two (Inst Inst) SkeletonInstSimplification)+(rule (replace_with_two first second)+ (SkeletonInstSimplification.ReplaceWithTwo first second))+(convert Inst SkeletonInstSimplification inst_to_skeleton_inst_simplification)(convert Value SkeletonInstSimplification value_to_skeleton_inst_simplification)diff --git a/cranelift/filetests/filetests/egraph/branch-to-trap.clif b/cranelift/filetests/filetests/egraph/branch-to-trap.clifnew file mode 100644index 0000000000..5a42db8468--- /dev/null+++ b/cranelift/filetests/filetests/egraph/branch-to-trap.clif@@ -0,0 +1,160 @@+test optimize precise-output+set opt_level=speed+target x86_64++;; The consequent (taken-if-true) successor is a just-trap block, so the `brif`+;; becomes a `trapnz` plus a `jump` to the surviving successor (carrying its+;; block arguments). The now-unreachable trap block is removed.+function %trapnz(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ brif v0, block1, block2(v1)+block1:+ trap user42+block2(v2: i64):+ return v2+}++; function %trapnz(i64, i64) -> i64 fast {+; block0(v0: i64, v1: i64):+; trapnz v0, user42+; jump block2+;+; block2:+; return v1+; }++;; The alternative (taken-if-false) successor is the just-trap block, so the+;; `brif` becomes a `trapz` plus a `jump` to the surviving successor.+function %trapz(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ brif v0, block1(v1), block2+block1(v2: i64):+ return v2+block2:+ trap user42+}++; function %trapz(i64, i64) -> i64 fast {+; block0(v0: i64, v1: i64):+; trapz v0, user42+; jump block1+;+; block1:+; return v1+; }++;; Both successors are just-trap blocks, so either rewrite is correct. Here the+;; alternative-successor rule wins, producing a `trapz` (trapping with the+;; alternative's code) and a jump to the still-trapping, still-reachable+;; consequent.+function %could_be_either_doesnt_matter(i64) -> i64 {+block0(v0: i64):+ brif v0, block1, block2+block1:+ trap user36+block2:+ trap user42+}++; function %could_be_either_doesnt_matter(i64) -> i64 fast {+; block0(v0: i64):+; trapz v0, user42+; jump block1+;+; block1:+; trap user36+; }++;; A block counts as a just-trap block even when it is not *only* a `trap`: a+;; pure (side-effect-free) prefix is allowed, because a trapping block never+;; returns so its pure instructions are unobservable. The `brif` is rewritten+;; and the pure prefix is dropped with the unreachable block.+function %trap_block_with_pure_prefix(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ brif v0, block1, block2(v1)+block1:+ v10 = iadd v0, v0+ v11 = imul v10, v10+ trap user42+block2(v2: i64):+ return v2+}++; function %trap_block_with_pure_prefix(i64, i64) -> i64 fast {+; block0(v0: i64, v1: i64):+; trapnz v0, user42+; jump block2+;+; block2:+; return v1+; }++;; A block with a *side-effecting* instruction before the `trap` is NOT a+;; just-trap block, so the `brif` is left alone.+function %trap_block_with_side_effect_not_rewritten(i64, i64) {+block0(v0: i64, v1: i64):+ brif v0, block1, block2+block1:+ store v1, v0+ trap user42+block2:+ return+}++; function %trap_block_with_side_effect_not_rewritten(i64, i64) fast {+; block0(v0: i64, v1: i64):+; brif v0, block1, block2+;+; block1:+; store.i64 v1, v0+; trap user42+;+; block2:+; return+; }++;; The `brif` folds into a `trapz v0, user1` that is identical to the `trapz`+;; already dominating it, so the newly inserted trap is re-processed and GVN'd+;; away (only one `trapz` remains).+function %folded_trap_is_gvned(i32) -> i32 {+block0(v0: i32):+ trapz v0, user1+ brif v0, block1, block2+block1:+ return v0+block2:+ trap user1+}++; function %folded_trap_is_gvned(i32) -> i32 fast {+; block0(v0: i32):+; trapz v0, user1+; jump block1+;+; block1:+; return v0+; }++;; The `brif`'s condition is a `uextend`, so after it folds into a `trapnz` the+;; newly inserted trap is re-processed and the truthiness-preserving `uextend`+;; is stripped from its condition (`trapnz v0` rather than `trapnz (uextend+;; v0)`).+function %folded_trap_condition_is_simplified(i8) -> i8 {+block0(v0: i8):+ v1 = uextend.i32 v0+ brif v1, block1, block2+block1:+ trap user1+block2:+ return v0+}++; function %folded_trap_condition_is_simplified(i8) -> i8 fast {+; block0(v0: i8):+; trapnz v0, user1+; jump block2+;+; block2:+; return v0+; }+diff --git a/cranelift/filetests/filetests/isa/x64/branches.clif b/cranelift/filetests/filetests/isa/x64/branches.clifindex da477c1989..bb2191bc31 100644--- a/cranelift/filetests/filetests/isa/x64/branches.clif+++ b/cranelift/filetests/filetests/isa/x64/branches.clif@@ -1,4 +1,5 @@test compile precise-output+set opt_level=speedtarget x86_64function %f0(i32, i32) -> i32 {@@ -1037,28 +1038,28 @@ block1(v5: i32):; pushq %rbp; movq %rsp, %rbp; block0:-; movl $0x1, %r11d-; movl $0x2, %edx-; movl $0x3, %ecx-; movl $0x4, %eax+; movl $0x4, %r11d+; movl $0x1, %edx+; movl $0x2, %ecx+; movl $0x3, %eax; movl $0x4, %r9d; movl %edi, %r10d; cmpl %r9d, %r10d; cmovbl %r10d, %r9d; br_table %r9, %rdi, %rsi; block1:+; movq %r11, %rax; jmp label6; block2:-; movq %r11, %rax+; movq %rdx, %rax; jmp label6; block3:-; movq %rdx, %rax+; movq %rcx, %rax; jmp label6; block4:-; movq %rdx, %rax+; movq %rcx, %rax; jmp label6; block5:-; movq %rcx, %rax; jmp label6; block6:; movq %rbp, %rsp@@ -1070,10 +1071,10 @@ block1(v5: i32):; pushq %rbp; movq %rsp, %rbp; block1: ; offset 0x4-; movl $1, %r11d-; movl $2, %edx-; movl $3, %ecx-; movl $4, %eax+; movl $4, %r11d+; movl $1, %edx+; movl $2, %ecx+; movl $3, %eax; movl $4, %r9d; movl %edi, %r10d; cmpl %r9d, %r10d@@ -1082,30 +1083,24 @@ block1(v5: i32):; movslq (%rdi, %r9, 4), %rsi; addq %rsi, %rdi; jmpq *%rdi-; sbbl %eax, (%rax)-; addb %al, (%rax)-; andl %eax, (%rax)+; sbbb $0, %al; addb %al, (%rax)-; subl %eax, (%rax)+; andb $0, %al; addb %al, (%rax)-; xorl %eax, (%rax)-; addb %al, (%rax)-; xorb $0, %al+; subb $0, %al; addb %al, (%rax); block2: ; offset 0x4d-; jmp 0x6d-; block3: ; offset 0x52; movq %r11, %rax-; jmp 0x6d-; block4: ; offset 0x5a-; movq %rdx, %rax-; jmp 0x6d-; block5: ; offset 0x62+; jmp 0x68+; block3: ; offset 0x55; movq %rdx, %rax-; jmp 0x6d-; block6: ; offset 0x6a+; jmp 0x68+; block4: ; offset 0x5d+; movq %rcx, %rax+; jmp 0x68+; block5: ; offset 0x65; movq %rcx, %rax-; block7: ; offset 0x6d+; block6: ; offset 0x68; movq %rbp, %rsp; popq %rbp; retqdiff --git a/cranelift/filetests/filetests/legalizer/branch-to-unconditional-trap.clif b/cranelift/filetests/filetests/legalizer/branch-to-unconditional-trap.clifdeleted file mode 100644index 8d742c63b1..0000000000--- a/cranelift/filetests/filetests/legalizer/branch-to-unconditional-trap.clif+++ /dev/null@@ -1,41 +0,0 @@-test legalizer-target aarch64--function %trapnz(i64, i64) -> i64 {-block0(v0: i64, v1: i64):- brif v0, block1, block2(v1)-block1:- trap user42-block2(v2: i64):- return v2-}--; check: block0(v0: i64, v1: i64):-; nextln: trapnz v0, user42-; nextln: jump block2(v1)--function %trapz(i64, i64) -> i64 {-block0(v0: i64, v1: i64):- brif v0, block1(v1), block2-block1(v2: i64):- return v2-block2:- trap user42-}--; check: block0(v0: i64, v1: i64):-; nextln: trapz v0, user42-; nextln: jump block1(v1)--function %could_be_either_doesnt_matter(i64) -> i64 {-block0(v0: i64):- brif v0, block1, block2-block1:- trap user36-block2:- trap user42-}--; check: block0(v0: i64):-; nextln: trapnz v0, user36-; nextln: jump block2diff --git a/cranelift/filetests/filetests/legalizer/bxor_imm.clif b/cranelift/filetests/filetests/legalizer/bxor_imm.clifdeleted file mode 100644index 4fc9a102f1..0000000000--- a/cranelift/filetests/filetests/legalizer/bxor_imm.clif+++ /dev/null@@ -1,13 +0,0 @@-test legalizer-target aarch64-target x86_64--function %foo(i64, i64) -> i64 {-block0(v0: i64, v1: i64):- v3 = iconst.i64 0x100000000- v2 = bxor v0, v3- return v2-}--; check: v3 = iconst.i64 0x0001_0000_0000-; check: v2 = bxor v0, v3diff --git a/cranelift/filetests/filetests/legalizer/iconst-i64.clif b/cranelift/filetests/filetests/legalizer/iconst-i64.clifdeleted file mode 100644index 359ca772e9..0000000000--- a/cranelift/filetests/filetests/legalizer/iconst-i64.clif+++ /dev/null@@ -1,13 +0,0 @@-test legalizer-target aarch64-target i686--function %foo() -> i64 {-block0:- v1 = iconst.i64 0x6400000042- return v1-}--; check: v2 = iconst.i32 66-; check: v3 = iconst.i32 100-; check: v1 = iconcat v2, v3diff --git a/cranelift/filetests/filetests/legalizer/isplit-bb.clif b/cranelift/filetests/filetests/legalizer/isplit-bb.clifdeleted file mode 100644index c89118cebc..0000000000--- a/cranelift/filetests/filetests/legalizer/isplit-bb.clif+++ /dev/null@@ -1,25 +0,0 @@-test legalizer-target aarch64-target x86_64--function u0:0(i128, i128, i64) -> i128 system_v {-block0(v0: i128, v1: i128, v2: i64):- jump block1--block1:- v17 = iadd v0, v1- v20 = iadd v1, v17- jump block79--block79:- v425 = iconst.i64 0- v875 = iconst.i64 1- v426 = icmp eq v425, v875- brif v426, block80, block85(v20, v17)--block80:- trap user1--block85(v462: i128, v874: i128):- trap user1-}diff --git a/cranelift/filetests/src/lib.rs b/cranelift/filetests/src/lib.rsindex 5443d31236..f124830988 100644--- a/cranelift/filetests/src/lib.rs+++ b/cranelift/filetests/src/lib.rs@@ -23,7 +23,6 @@ mod test_compile;mod test_domtree;mod test_inline;mod test_interpret;-mod test_legalizer;mod test_optimize;mod test_print_cfg;mod test_run;@@ -92,7 +91,6 @@ fn new_subtest(parsed: &TestCommand) -> anyhow::Result<Box<dyn subtest::SubTest>"domtree" => test_domtree::subtest(parsed),"inline" => test_inline::subtest(parsed),"interpret" => test_interpret::subtest(parsed),- "legalizer" => test_legalizer::subtest(parsed),"optimize" => test_optimize::subtest(parsed),"print-cfg" => test_print_cfg::subtest(parsed),"run" => test_run::subtest(parsed),diff --git a/cranelift/filetests/src/test_inline.rs b/cranelift/filetests/src/test_inline.rsindex 68df7089cc..183ab875cb 100644--- a/cranelift/filetests/src/test_inline.rs+++ b/cranelift/filetests/src/test_inline.rs@@ -67,13 +67,8 @@ impl SubTest for TestInline {}fn run(&self, func: Cow<ir::Function>, context: &Context) -> Result<()> {- // Legalize this function.let isa = context.isa.unwrap();let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned());- comp_ctx- .legalize(isa)- .map_err(|e| crate::pretty_anyhow_error(&comp_ctx.func, e))- .context("error while legalizing")?;// Insert this function in our map for inlining into subsequent// functions.diff --git a/cranelift/filetests/src/test_legalizer.rs b/cranelift/filetests/src/test_legalizer.rsdeleted file mode 100644index 77b33b4c87..0000000000--- a/cranelift/filetests/src/test_legalizer.rs+++ /dev/null@@ -1,46 +0,0 @@-//! Test command for checking the IR legalizer.-//!-//! The `test legalizer` test command runs each function through `legalize_function()` and sends-//! the result to filecheck.--use crate::subtest::{Context, SubTest, run_filecheck};-use cranelift_codegen::ir::Function;-use cranelift_reader::TestCommand;-use std::borrow::Cow;--struct TestLegalizer;--pub fn subtest(parsed: &TestCommand) -> anyhow::Result<Box<dyn SubTest>> {- assert_eq!(parsed.command, "legalizer");- if !parsed.options.is_empty() {- anyhow::bail!("No options allowed on {parsed}");- }- Ok(Box::new(TestLegalizer))-}--impl SubTest for TestLegalizer {- fn name(&self) -> &'static str {- "legalizer"- }-- fn is_mutating(&self) -> bool {- true- }-- fn needs_isa(&self) -> bool {- true- }-- fn run(&self, func: Cow<Function>, context: &Context) -> anyhow::Result<()> {- let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned());- let isa = context.isa.expect("legalizer needs an ISA");-- comp_ctx.compute_cfg();- comp_ctx- .legalize(isa)- .map_err(|e| crate::pretty_anyhow_error(&comp_ctx.func, e))?;-- let text = comp_ctx.func.display().to_string();- run_filecheck(&text, context)- }-}diff --git a/cranelift/filetests/src/test_safepoint.rs b/cranelift/filetests/src/test_safepoint.rsindex 955147c651..5432459370 100644--- a/cranelift/filetests/src/test_safepoint.rs+++ b/cranelift/filetests/src/test_safepoint.rs@@ -21,11 +21,7 @@ impl SubTest for TestSafepoint {fn run(&self, func: Cow<Function>, context: &Context) -> anyhow::Result<()> {let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned());- let isa = context.isa.expect("register allocator needs an ISA");comp_ctx.compute_cfg();- comp_ctx- .legalize(isa)- .map_err(|e| crate::pretty_anyhow_error(&comp_ctx.func, e))?;comp_ctx.compute_domtree();let text = comp_ctx.func.display().to_string();diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex e22914e125..793cd20664 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -40,7 +40,7 @@ use wasmtime_environ::obj::{ELF_WASMTIME_EXCEPTIONS, ELF_WASMTIME_FRAMES};use wasmtime_environ::{Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,- FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, GetPtrSize, HostCall, Inlining,+ FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, GetPtrSize, HostCall,InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize,StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables,WasmFuncType, WasmValType, prelude::*,@@ -561,14 +561,6 @@ impl wasmtime_environ::Compiler for Compiler {&mut func_env,)?;- if self.tunables.inlining != Inlining::No {- compiler- .cx- .codegen_context- .legalize(isa)- .map_err(|e| CompileError::Codegen(e.to_string()))?;- }-let needs_gc_heap = func_env.needs_gc_heap();if let Some((_, slot_builder)) = func_env.state_slot {diff --git a/tests/disas/arith.wat b/tests/disas/arith.watindex 26db3991b3..dce925c702 100644--- a/tests/disas/arith.wat+++ b/tests/disas/arith.wat@@ -27,8 +27,10 @@;; @0021 v3 = iconst.i32 4;; @0023 v4 = iconst.i32 4;; @0025 v5 = isub v3, v4 ; v3 = 4, v4 = 4-;; @002c trapnz v5, user12-;; @002a jump block4+;; @002a brif v5, block2, block4+;;+;; block2:+;; @002c trap user12;;;; block4:;; @002e v6 = iconst.i32 6diff --git a/tests/disas/conditional-traps.wat b/tests/disas/conditional-traps.watindex b681204871..6927686c71 100644--- a/tests/disas/conditional-traps.wat+++ b/tests/disas/conditional-traps.wat@@ -30,7 +30,7 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):-;; @0031 trapnz v2, user12+;; @002f trapnz v2, user12;; @002f jump block3;;;; block3:@@ -51,7 +51,7 @@;; block0(v0: i64, v1: i64, v2: i32):;; @0038 v3 = iconst.i32 0;; @0038 v4 = icmp eq v2, v3 ; v3 = 0-;; @003b trapnz v4, user12+;; @0039 trapnz v4, user12;; @0039 jump block3;;;; block3:diff --git a/tests/disas/gc/drc/struct-new-default.wat b/tests/disas/gc/drc/struct-new-default.watindex 180e42b3f9..500ad58e94 100644--- a/tests/disas/gc/drc/struct-new-default.wat+++ b/tests/disas/gc/drc/struct-new-default.wat@@ -44,7 +44,7 @@;; @0021 v18 = iconst.i64 28;; @0021 v19 = iadd v15, v18 ; v18 = 28;; @0021 istore8 user2 little region4 v4, v19 ; v4 = 0-;; jump block3+;; @0021 jump block3;;;; block3:;; v60 = iconst.i32 0diff --git a/tests/disas/if-reachability-translation-2.wat b/tests/disas/if-reachability-translation-2.watindex 6e7c3dce31..d0537025ed 100644--- a/tests/disas/if-reachability-translation-2.wat+++ b/tests/disas/if-reachability-translation-2.wat@@ -22,8 +22,10 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):-;; @001d trapnz v2, user12-;; @001b jump block4+;; @001b brif v2, block2, block4+;;+;; block2:+;; @001d trap user12;;;; block4:;; @0020 jump block3diff --git a/tests/disas/if-reachability-translation-3.wat b/tests/disas/if-reachability-translation-3.watindex c5da264d4a..0cf8f6ff4b 100644--- a/tests/disas/if-reachability-translation-3.wat+++ b/tests/disas/if-reachability-translation-3.wat@@ -22,11 +22,13 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):-;; @001f trapz v2, user12-;; @001b jump block2+;; @001b brif v2, block2, block4;;;; block2:-;; jump block3+;; @001e jump block3+;;+;; block4:+;; @001f trap user12;;;; block3:;; @0021 v4 = iconst.i32 0diff --git a/tests/disas/if-reachability-translation-4.wat b/tests/disas/if-reachability-translation-4.watindex defdc3dbdd..4f3522c9aa 100644--- a/tests/disas/if-reachability-translation-4.wat+++ b/tests/disas/if-reachability-translation-4.wat@@ -22,8 +22,10 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32):-;; @001d trapnz v2, user12-;; @001b jump block4+;; @001b brif v2, block2, block4+;;+;; block2:+;; @001d trap user12;;;; block4:;; @001f trap user12diff --git a/tests/disas/if-reachability-translation-5.wat b/tests/disas/if-reachability-translation-5.watindex e74ca134bd..9bb409436f 100644--- a/tests/disas/if-reachability-translation-5.wat+++ b/tests/disas/if-reachability-translation-5.wat@@ -24,12 +24,16 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32):-;; @0024 trapz v2, user12-;; @001c jump block2+;; @001c brif v2, block2, block5;;;; block2:-;; @0022 trapz.i32 v3, user12-;; @0020 jump block3+;; @0020 brif.i32 v3, block3, block4+;;+;; block4:+;; @0022 trap user12+;;+;; block5:+;; @0024 trap user12;;;; block3:;; @0026 v5 = iconst.i32 0diff --git a/tests/disas/if-reachability-translation-6.wat b/tests/disas/if-reachability-translation-6.watindex d43fe819d4..2b3ba1c7de 100644--- a/tests/disas/if-reachability-translation-6.wat+++ b/tests/disas/if-reachability-translation-6.wat@@ -24,12 +24,16 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32):-;; @001e trapnz v2, user12-;; @001c jump block4+;; @001c brif v2, block2, block4+;;+;; block2:+;; @001e trap user12;;;; block4:-;; @0024 trapz.i32 v3, user12-;; @0022 jump block3+;; @0022 brif.i32 v3, block3, block5+;;+;; block5:+;; @0024 trap user12;;;; block3:;; @0026 v5 = iconst.i32 0diff --git a/tests/disas/if-unreachable-else-params-2.wat b/tests/disas/if-unreachable-else-params-2.watindex eef46daa01..349794c2d8 100644--- a/tests/disas/if-unreachable-else-params-2.wat+++ b/tests/disas/if-unreachable-else-params-2.wat@@ -32,8 +32,7 @@;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32):;; @0049 v5 = f64const 0x1.0000000000000p0-;; @005d trapz v3, user12-;; @0056 jump block2+;; @0056 brif v3, block2, block4;;;; block2:;; @0058 v7 = uextend.i64 v2@@ -42,6 +41,9 @@;; @0058 v10 = sload16.i64 little region4 v9;; @005c jump block3;;+;; block4:+;; @005d trap user12+;;;; block3:;; @005f jump block1;;diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.watindex cf307d21cc..06d2b6e585 100644--- a/tests/disas/stack-switching/symmetric-switch.wat+++ b/tests/disas/stack-switching/symmetric-switch.wat@@ -76,8 +76,7 @@;; block2(v28: i64, v29: i64):;; @003e v30 = iconst.i64 1;; @003e v31 = icmp eq v28, v30 ; v30 = 1-;; @003e trapnz v31, user22-;; @003e jump block3+;; @003e brif v31, block7, block3;;;; block3:;; @003e v32 = load.i64 notrap aligned v29+48@@ -104,6 +103,9 @@;; @003e v48 = iadd.i32 v39, v47 ; v47 = 1;; @003e brif v46, block6, block4(v48);;+;; block7 cold:+;; @003e trap user22+;;;; block6:;; @003e store.i64 notrap aligned v29, v27+64;; @003e v49 = iconst.i64 120