A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
aarch64: lower `ishl` + `ushr`/`sshr` pairs to a single `ubfm`/`sbfm` instruction (#14187)
Alexander Raffertybe1f8f776cfd
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex f244f696b8..bfbcf423f0 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -209,6 +209,27 @@(from_bits u8)(to_bits u8))+ ;; A bitfield move instruction, either UBFM or SBFM.+ ;; The BFM instruction is instead encoded as BitfieldMoveMod.+ (BitfieldMove+ (size OperandSize)+ (bfm_op BfmOp)+ (rd WritableReg)+ (rn Reg)+ (immr UImm6)+ (imms UImm6))++ ;; The bitfield move instruction that modifies its `rd`, i.e. BFM.+ ;; The input state of `rd` is represented as `ri` (for "input"),+ ;; and is constrained with `rd` so they use the same register.+ (BitfieldMoveMod+ (size OperandSize)+ (rd WritableReg)+ (ri Reg)+ (rn Reg)+ (immr UImm6)+ (imms UImm6))+;; A conditional-select operation.(CSel(rd WritableReg)@@ -1208,6 +1229,7 @@;; Leaves currently without specs: excluded from verification via todo.(attr MInst.Udf (tag TODO))+(attr MInst.BitfieldMove (tag TODO));; An ALU operation. This can be paired with several instruction formats;; below (see `Inst`) in any combination.@@ -1267,6 +1289,14 @@(MovN)))+;; A bitfield move operation.+;; Note that BFM is excluded as it modifies rather than overwrites `rd`.+(type BfmOp+ (enum+ (UBfm)+ (SBfm)+))+(model UImm5 (type (bv 5)))(type UImm5 (primitive UImm5))@@ -1286,6 +1316,9 @@(model ImmShift (type (bv 6)))(type ImmShift (primitive ImmShift))+(model UImm6 (type (bv 6)))+(type UImm6 (primitive UImm6))+(model ShiftOpAndAmt(type(struct@@ -2959,6 +2992,14 @@(_ Unit (emit (MInst.Extend dst rn signed from_bits to_bits))))dst))+;; Helper for emitting `MInst.BitfieldMove` instructions.+(attr bitfield_move (veri chain))+(decl bitfield_move (Type BfmOp Reg UImm6 UImm6) Reg)+(rule (bitfield_move ty bfm_op rn immr imms)+ (let ((dst WritableReg (temp_writable_reg ty))+ (_ Unit (emit (MInst.BitfieldMove (operand_size ty) bfm_op dst rn immr imms))))+ dst))+;; Helper for emitting `MInst.FpuExtend` instructions.(attr fpu_extend (veri chain))(decl fpu_extend (Reg ScalarSize) Reg)diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex ec09d6d156..66d6cd1fe0 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -417,6 +417,15 @@ fn enc_ccmp_imm(size: OperandSize, rn: Reg, imm: UImm5, nzcv: NZCV, cond: Cond)| nzcv.bits()}+impl BfmOp {+ fn opc(self) -> u8 {+ match self {+ BfmOp::UBfm => 0b10,+ BfmOp::SBfm => 0b00,+ }+ }+}+fn enc_bfm(opc: u8, size: OperandSize, rd: Writable<Reg>, rn: Reg, immr: u8, imms: u8) -> u32 {match size {OperandSize::Size64 => {@@ -2954,13 +2963,36 @@ impl MachInstEmit for Inst {from_bits,to_bits,} => {- let (opc, size) = if signed {- (0b00, OperandSize::from_bits(to_bits))+ let (bfm_op, size) = if signed {+ (BfmOp::SBfm, OperandSize::from_bits(to_bits))} else {- (0b10, OperandSize::Size32)+ (BfmOp::UBfm, OperandSize::Size32)};+ let opc = bfm_op.opc();sink.put4(enc_bfm(opc, size, rd, rn, 0, from_bits - 1));}+ &Inst::BitfieldMove {+ size,+ bfm_op,+ rd,+ rn,+ immr,+ imms,+ } => {+ let opc = bfm_op.opc();+ sink.put4(enc_bfm(opc, size, rd, rn, immr.value(), imms.value()));+ }+ &Inst::BitfieldMoveMod {+ size,+ rd,+ ri,+ rn,+ immr,+ imms,+ } => {+ debug_assert_eq!(rd.to_reg(), ri);+ sink.put4(enc_bfm(0b01, size, rd, rn, immr.value(), imms.value()));+ }&Inst::Jump { ref dest } => {let off = sink.cur_offset();// Indicate that the jump uses a label, if so, so that a fixup can occur later.diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rsindex 72e26095e2..b5c72cb4a2 100644--- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs@@ -547,11 +547,26 @@ pub struct ImmShift {impl ImmShift {/// Create an ImmShift from raw bits, if possible.pub fn maybe_from_u64(val: u64) -> Option<ImmShift> {- if val < 64 {- Some(ImmShift { imm: val as u8 })- } else {- None- }+ (val < 64).then_some(ImmShift { imm: val as u8 })+ }++ /// Get the immediate value.+ pub fn value(&self) -> u8 {+ self.imm+ }+}++/// A 6-bit immediate used by the `immr` and `imms` fields of bitfield move instructions.+#[derive(Copy, Clone, Debug)]+pub struct UImm6 {+ /// 6-bit immediate.+ pub imm: u8,+}++impl UImm6 {+ /// Create a UImm6 from raw bits, if possible.+ pub fn maybe_from_u8(val: u8) -> Option<UImm6> {+ (val < 64).then_some(UImm6 { imm: val })}/// Get the immediate value.@@ -915,6 +930,12 @@ impl PrettyPrint for ImmShift {}}+impl PrettyPrint for UImm6 {+ fn pretty_print(&self, _: u8) -> String {+ format!("#{}", self.imm)+ }+}+impl PrettyPrint for MoveWideConst {fn pretty_print(&self, _: u8) -> String {if self.shift == 0 {diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex 9598edffee..230cc2abb4 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -34,10 +34,10 @@ mod emit_tests;// Instructions (top level): definitionpub use crate::isa::aarch64::lower::isle::generated_code::{- ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, BranchTargetType, FPUOp1,- FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUModOp,- VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, VecRRNarrowOp,- VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,+ ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BfmOp, BitOp, BranchTargetType,+ FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp,+ VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp,+ VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,};/// A floating-point unit (FPU) operation with two args, a register and an immediate.@@ -60,6 +60,16 @@ pub enum FPUOpRIMod {Sli64(FPULeftShiftImm),}+impl BfmOp {+ /// Get the assembly mnemonic for this opcode.+ pub fn op_str(&self) -> &'static str {+ match self {+ BfmOp::UBfm => "ubfm",+ BfmOp::SBfm => "sbfm",+ }+ }+}+impl BitOp {/// Get the assembly mnemonic for this opcode.pub fn op_str(&self) -> &'static str {@@ -812,6 +822,17 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_def(rd);collector.reg_use(rn);}+ Inst::BitfieldMove { rd, rn, .. } => {+ // The UBFM and SBFM instructions overwrite all bits in `rd`,+ // unlike BFM which is represented as `BitfieldMoveMod` instead.+ collector.reg_def(rd);+ collector.reg_use(rn);+ }+ Inst::BitfieldMoveMod { rd, ri, rn, .. } => {+ collector.reg_reuse_def(rd, 1); // `rd` == `ri`.+ collector.reg_use(ri);+ collector.reg_use(rn);+ }Inst::Args { args } => {for ArgPair { vreg, preg } in args {collector.reg_fixed_def(vreg, *preg);@@ -2637,6 +2658,36 @@ impl Inst {format!("{op} {rd}, {rn}")}}+ &Inst::BitfieldMove {+ size,+ bfm_op,+ rd,+ rn,+ immr,+ imms,+ } => {+ let op = bfm_op.op_str();+ let rd = pretty_print_ireg(rd.to_reg(), size);+ let rn = pretty_print_ireg(rn, size);+ let immr = immr.pretty_print(0);+ let imms = imms.pretty_print(0);+ format!("{op} {rd}, {rn}, {immr}, {imms}")+ }+ &Inst::BitfieldMoveMod {+ size,+ rd,+ ri,+ rn,+ immr,+ imms,+ } => {+ let rd = pretty_print_ireg(rd.to_reg(), size);+ let ri = pretty_print_ireg(ri, size);+ let rn = pretty_print_ireg(rn, size);+ let immr = immr.pretty_print(0);+ let imms = imms.pretty_print(0);+ format!("bfm {rd}, {ri}, {rn}, {immr}, {imms}")+ }&Inst::Call { ref info } => {let try_call = info.try_call_infodiff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex 53abae090a..e713d3c9e2 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -1699,6 +1699,23 @@(rule sshr_64 (lower (sshr $I64 x y))(do_shift (ALUOp.Asr) $I64 (put_in_reg_sext64 x) y))+;; Specialized lowerings to generate a single `ubfm`/`sbfm` instruction from+;; an appropriate pair of `ishl` and `ushr`/`sshr` operations.+(rule sbfm 1 (lower+ (sshr (ty_32_or_64 ty) (ishl _ x (u64_from_iconst a)) (u64_from_iconst b)))+ (bitfield_move ty (BfmOp.SBfm) x (bfm_immr ty a b) (bfm_imms ty a b)))+(rule ubfm 1 (lower+ (ushr (ty_32_or_64 ty) (ishl _ x (u64_from_iconst a)) (u64_from_iconst b)))+ (bitfield_move ty (BfmOp.UBfm) x (bfm_immr ty a b) (bfm_imms ty a b)))++;; Helper methods for constructing the correct `immr` and `imms` immediates.+(spec (bfm_immr ty a b) (provide true))+(decl bfm_immr (Type u64 u64) UImm6)+(extern constructor bfm_immr bfm_immr)+(spec (bfm_imms ty a b) (provide true))+(decl bfm_imms (Type u64 u64) UImm6)+(extern constructor bfm_imms bfm_imms)+;; Shift for i128.(rule (lower (sshr $I128 x y))(lower_sshr128 x (value_regs_get y 0)))diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rsindex 7806c4cd8d..a2abe420a3 100644--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs@@ -9,7 +9,7 @@ use super::{ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI,FPUOpRIMod, FloatCC, Imm12, ImmLogic, ImmShift, Inst as MInst, IntCC, MachLabel, MemLabel,MoveWideConst, MoveWideOp, NZCV, Opcode, OperandSize, Reg, SImm9, ScalarSize, ShiftOpAndAmt,- UImm5, UImm12Scaled, VecMisc2, VectorSize, fp_reg, lower_condcode, stack_reg,+ UImm5, UImm6, UImm12Scaled, VecMisc2, VectorSize, fp_reg, lower_condcode, stack_reg,writable_link_reg, writable_zero_reg, zero_reg,};use crate::ir::{ArgumentExtension, condcodes};@@ -265,6 +265,29 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {ImmShift::maybe_from_u64(n.into()).unwrap()}+ /// Compute the `immr` value for an `sbfm` instruction,+ /// derived by fusing an `ishl` by amount `a`, with an `sshr` by amount `b`.+ fn bfm_immr(&mut self, ty: Type, a: u64, b: u64) -> UImm6 {+ let w = ty.lane_bits() as u8;+ debug_assert!(w <= 64);++ let a = (a as u8) & (w - 1);+ let b = (b as u8) & (w - 1);+ let result = if a <= b { b - a } else { w - (a - b) };+ UImm6::maybe_from_u8(result).expect("result is always less than 64")+ }++ /// Compute the `imms` value for an `sbfm` instruction,+ /// derived by fusing an `ishl` by amount `a`, with an `sshr` by amount `b`.+ fn bfm_imms(&mut self, ty: Type, a: u64, _b: u64) -> UImm6 {+ let w = ty.lane_bits() as u8;+ debug_assert!(w <= 64);++ let a = (a as u8) & (w - 1);+ let result = w - 1 - (a & (w - 1));+ UImm6::maybe_from_u8(result).expect("result is always less than 64")+ }+fn lshr_from_u64(&mut self, ty: Type, n: u64) -> Option<ShiftOpAndAmt> {let shiftimm = ShiftOpShiftImm::maybe_from_shift(n)?;if let Ok(bits) = u8::try_from(ty_bits(ty)) {diff --git a/cranelift/filetests/filetests/isa/aarch64/shift-op.clif b/cranelift/filetests/filetests/isa/aarch64/shift-op.clifindex d0c0c972b0..56e78b7c08 100644--- a/cranelift/filetests/filetests/isa/aarch64/shift-op.clif+++ b/cranelift/filetests/filetests/isa/aarch64/shift-op.clif@@ -36,4 +36,3 @@ block0(v0: i32):; block0: ; offset 0x0; lsl w0, w0, #0x15; ret-diff --git a/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clif b/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clifindex 42d066572a..dcd267713a 100644--- a/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clif+++ b/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clif@@ -666,3 +666,139 @@ block0(v0: i64):; lsl x0, x0, #0x11; ret+function %f28(i64) -> i64 {+block0(v0: i64):+ v1 = iconst.i32 32+ v2 = ishl.i64 v0, v1+ v3 = iconst.i32 52+ v4 = sshr.i64 v2, v3+ return v4+}++; VCode:+; block0:+; sbfm x0, x0, #20, #31+; ret+;+; Disassembled:+; block0: ; offset 0x0+; sbfx x0, x0, #0x14, #0xc+; ret++function %f29(i32) -> i32 {+block0(v0: i32):+ v1 = iconst.i32 16+ v2 = ishl.i32 v0, v1+ v3 = iconst.i32 26+ v4 = sshr.i32 v2, v3+ return v4+}++; VCode:+; block0:+; sbfm w0, w0, #10, #15+; ret+;+; Disassembled:+; block0: ; offset 0x0+; sbfx w0, w0, #0xa, #6+; ret++function %f30(i64) -> i64 {+block0(v0: i64):+ v1 = iconst.i32 40+ v2 = ishl.i64 v0, v1+ v3 = iconst.i32 20+ v4 = sshr.i64 v2, v3+ return v4+}++; VCode:+; block0:+; sbfm x0, x0, #44, #23+; ret+;+; Disassembled:+; block0: ; offset 0x0+; sbfiz x0, x0, #0x14, #0x18+; ret++function %f31(i32) -> i32 {+block0(v0: i32):+ v1 = iconst.i32 12+ v2 = ishl.i32 v0, v1+ v3 = iconst.i32 8+ v4 = sshr.i32 v2, v3+ return v4+}++; VCode:+; block0:+; sbfm w0, w0, #28, #19+; ret+;+; Disassembled:+; block0: ; offset 0x0+; sbfiz w0, w0, #4, #0x14+; ret++function %f32(i64) -> i64 {+block0(v0: i64):+ v1 = iconst.i32 12+ v2 = ishl.i64 v0, v1+ v3 = iconst.i32 20+ v4 = sshr.i64 v2, v3+ v5 = iadd.i64 v2, v4+ return v5+}++; VCode:+; block0:+; sbfm x3, x0, #8, #51+; add x0, x3, x0, LSL 12+; ret+;+; Disassembled:+; block0: ; offset 0x0+; sbfx x3, x0, #8, #0x2c+; add x0, x3, x0, lsl #12+; ret++function %f33(i64) -> i64 {+block0(v0: i64):+ v1 = iconst.i32 48+ v2 = ishl.i64 v0, v1+ v3 = iconst.i32 54+ v4 = ushr.i64 v2, v3+ return v4+}++; VCode:+; block0:+; ubfm x0, x0, #6, #15+; ret+;+; Disassembled:+; block0: ; offset 0x0+; ubfx x0, x0, #6, #0xa+; ret++function %f34(i64) -> i64 {+block0(v0: i64):+ v1 = iconst.i32 48+ v2 = ishl.i64 v0, v1+ v3 = iconst.i32 32+ v4 = ushr.i64 v2, v3+ return v4+}++; VCode:+; block0:+; ubfm x0, x0, #48, #15+; ret+;+; Disassembled:+; block0: ; offset 0x0+; ubfiz x0, x0, #0x10, #0x10+; ret+