A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
aarch64: lower mixed unsigned-by-signed i8 dot to USDOT (FEAT_I8MM) (#14054)
Damilare Darmie Akinlajae8ac8c27f199
diff --git a/cranelift/codegen/meta/src/isa/arm64.rs b/cranelift/codegen/meta/src/isa/arm64.rsindex 1f5f0de237..968ac6e14f 100644--- a/cranelift/codegen/meta/src/isa/arm64.rs+++ b/cranelift/codegen/meta/src/isa/arm64.rs@@ -32,6 +32,14 @@ pub(crate) fn define() -> TargetIsa {"",false,);+ settings.add_bool(+ "has_i8mm",+ "Has Int8 Matrix Multiplication (FEAT_I8MM) support; enables lowering \+ the mixed unsigned-by-signed i8 dot product to USDOT instead of a \+ umull/smull + saddlp widening fallback.",+ "",+ false,+ );settings.add_bool("sign_return_address_all","If function return address signing is enabled, then apply it to all \diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex 893b2a61a0..6bae128cec 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -1853,6 +1853,8 @@(Fmls);; Signed integer dot product (FEAT_DotProd)(Sdot)+ ;; Unsigned-by-signed integer dot product (FEAT_I8MM)+ (Usdot)));; A Vector miscellaneous operation with two registers.@@ -2076,6 +2078,10 @@(decl use_dotprod () Inst)(extern extractor use_dotprod use_dotprod)+;; Matches any instruction when the `has_i8mm` (FEAT_I8MM) setting is on.+(decl use_i8mm () Inst)+(extern extractor use_i8mm use_i8mm)+(spec (use_fp16) (provide (not result)))(decl pure use_fp16 () bool)(extern constructor use_fp16 use_fp16)@@ -3503,6 +3509,13 @@(rule (sdot acc a b)(vec_rrr_mod (VecALUModOp.Sdot) acc a b (VectorSize.Size32x4)))+;; USDOT Vd.4S, Vn.16B, Vm.16B -- like `sdot`, but the first source is read as+;; unsigned bytes and the second as signed. `acc` is both source and+;; destination; `a` (unsigned) / `b` (signed) are i8x16.+(decl usdot (Reg Reg Reg) Reg)+(rule (usdot acc a b)+ (vec_rrr_mod (VecALUModOp.Usdot) acc a b (VectorSize.Size32x4)))+;; Helper for generating a `udf` instruction.(decl udf (TrapCode) SideEffectNoResult)diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex 0c261c267e..b10e412763 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -2769,6 +2769,9 @@ impl MachInstEmit for Inst {// so it is baked into top11; only Q (from `size`) is variable.// top11 (Q=0) | q<<9 with bit15_10 yields 0x4E809400 for .4S/.16B.VecALUModOp::Sdot => (0b000_01110_10_0, 0b100101),+ // USDOT Vd.4S, Vn.16B, Vm.16B (FEAT_I8MM). Same shape as+ // SDOT; only the opcode field differs.+ VecALUModOp::Usdot => (0b000_01110_10_0, 0b100111),};sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));}diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex acaeb8a431..81ddc33554 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -2290,6 +2290,7 @@ impl Inst {// Note: the real operand arrangement is .4s, .16b, .16b;// this debug print renders all lanes as .4s.VecALUModOp::Sdot => ("sdot", VectorSize::Size32x4),+ VecALUModOp::Usdot => ("usdot", VectorSize::Size32x4),};let rd = pretty_print_vreg_vector(rd.to_reg(), size);let ri = pretty_print_vreg_vector(ri, size);diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex 4b4dec7524..bf364b7340 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -412,6 +412,23 @@c))))(sdot c a b))+;; With FEAT_I8MM, fold the same tree in its mixed unsigned-by-signed form+;; into a single `usdot`. This is the shape quantized inference produces —+;; unsigned activations against signed weights — and it differs from the+;; `sdot` case above only in that one operand widens unsigned.+(rule 9 (lower (and (use_i8mm)+ (has_type $I32X4+ (iadd _+ (iadd_pairwise _+ (iadd_pairwise _+ (swiden_low _ lo @ (imul _ (uwiden_low _ a) (swiden_low _ b)))+ (swiden_high _ lo))+ (iadd_pairwise _+ (swiden_low _ hi @ (imul _ (uwiden_high _ a) (swiden_high _ b)))+ (swiden_high _ hi)))+ c))))+ (usdot c a b))+;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule -1 (lower (iabs ty @ (multi_lane _ _) x))diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rsindex dde8371da3..d50940b2ee 100644--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs@@ -187,6 +187,14 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {}}+ fn use_i8mm(&mut self, _: Inst) -> Option<()> {+ if self.backend.isa_flags.has_i8mm() {+ Some(())+ } else {+ None+ }+ }+fn use_fp16(&mut self) -> bool {self.backend.isa_flags.has_fp16()}diff --git a/cranelift/filetests/filetests/isa/aarch64/usdot.clif b/cranelift/filetests/filetests/isa/aarch64/usdot.clifnew file mode 100644index 0000000000..c9dba6769d--- /dev/null+++ b/cranelift/filetests/filetests/isa/aarch64/usdot.clif@@ -0,0 +1,35 @@+test compile precise-output+set unwind_info=false+target aarch64 has_i8mm++;; The mixed unsigned-by-signed i8 dot tree folds into a single `usdot`:+;; `a` widens unsigned, `b` widens signed.+function %usdot_i8x16(i32x4, i8x16, i8x16) -> i32x4 {+block0(v0: i32x4, v1: i8x16, v2: i8x16):+ v3 = uwiden_low v1+ v4 = swiden_low v2+ v5 = imul v3, v4+ v6 = swiden_low v5+ v7 = swiden_high v5+ v8 = iadd_pairwise v6, v7+ v9 = uwiden_high v1+ v10 = swiden_high v2+ v11 = imul v9, v10+ v12 = swiden_low v11+ v13 = swiden_high v11+ v14 = iadd_pairwise v12, v13+ v15 = iadd_pairwise v8, v14+ v16 = iadd v15, v0+ return v16+}++; VCode:+; block0:+; usdot v0.4s, v0.4s, v1.4s, v2.4s+; ret+;+; Disassembled:+; block0: ; offset 0x0+; usdot v0.4s, v1.16b, v2.16b+; ret+diff --git a/cranelift/filetests/filetests/runtests/usdot.clif b/cranelift/filetests/filetests/runtests/usdot.clifnew file mode 100644index 0000000000..a1060234c0--- /dev/null+++ b/cranelift/filetests/filetests/runtests/usdot.clif@@ -0,0 +1,50 @@+; Execution coverage for the mixed unsigned-by-signed i8 dot product.+; The lowering only changes aarch64 codegen; running on the interpreter and+; every native target confirms the result is unchanged either way.+;+; The lanes are summed to a scalar before returning: a vector return does+; not fit in registers on every target, and the per-lane result is already+; pinned by the precise-output test in isa/aarch64.+;+; riscv64 is not listed: its vector backend has no lowering for the lane+; extraction this reduction needs (and no vector return either), both+; unrelated to the dot itself. The interpreter plus the three remaining+; targets cover the semantics.+test interpret+test run+target aarch64+target aarch64 has_i8mm+target x86_64+target s390x++function %usdot_mixed(i32x4, i8x16, i8x16) -> i32 {+block0(v0: i32x4, v1: i8x16, v2: i8x16):+ v3 = uwiden_low v1+ v4 = swiden_low v2+ v5 = imul v3, v4+ v6 = swiden_low v5+ v7 = swiden_high v5+ v8 = iadd_pairwise v6, v7+ v9 = uwiden_high v1+ v10 = swiden_high v2+ v11 = imul v9, v10+ v12 = swiden_low v11+ v13 = swiden_high v11+ v14 = iadd_pairwise v12, v13+ v15 = iadd_pairwise v8, v14+ v16 = iadd v15, v0+ v17 = extractlane v16, 0+ v18 = extractlane v16, 1+ v19 = extractlane v16, 2+ v20 = extractlane v16, 3+ v21 = iadd v17, v18+ v22 = iadd v19, v20+ v23 = iadd v21, v22+ return v23+}+; 200 read as u8 is 200, not -56 — a signed reading would give a different+; total, so this pins the unsigned widening.+; lanes: 4*(200*-1) + 4*(1*3) + 4*(0*5) + 4*(2*-2) = -800 + 12 + 0 - 16+; run: %usdot_mixed([0 0 0 0], [200 200 200 200 1 1 1 1 0 0 0 0 2 2 2 2], [-1 -1 -1 -1 3 3 3 3 5 5 5 5 -2 -2 -2 -2]) == -804+; acc [1 2 3 4] plus 4 per lane = [5 6 7 8]+; run: %usdot_mixed([1 2 3 4], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1]) == 26diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rsindex bffce9ab73..bfcdbf3088 100644--- a/cranelift/native/src/lib.rs+++ b/cranelift/native/src/lib.rs@@ -126,6 +126,10 @@ pub fn infer_native_flags(isa_builder: &mut dyn Configurable) -> Result<(), &'stisa_builder.enable("has_dotprod").unwrap();}+ if std::arch::is_aarch64_feature_detected!("i8mm") {+ isa_builder.enable("has_i8mm").unwrap();+ }+if cfg!(target_os = "macos") {// Pointer authentication is always available on Apple Silicon.isa_builder.enable("sign_return_address").unwrap();diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex 1ceb91a4ac..6534da5b5e 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -4681,6 +4681,7 @@ fn detect_host_feature(feature: &str) -> Option<bool> {"paca" => Some(std::arch::is_aarch64_feature_detected!("paca")),"fp16" => Some(std::arch::is_aarch64_feature_detected!("fp16")),"dotprod" => Some(std::arch::is_aarch64_feature_detected!("dotprod")),+ "i8mm" => Some(std::arch::is_aarch64_feature_detected!("i8mm")),_ => None,};diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rsindex 7c8acbf735..87b7cb4351 100644--- a/crates/wasmtime/src/engine.rs+++ b/crates/wasmtime/src/engine.rs@@ -571,6 +571,7 @@ information about this check\"has_pauth" => "paca","has_fp16" => "fp16","has_dotprod" => "dotprod",+ "has_i8mm" => "i8mm",// aarch64 features which don't need detection// No effect on its own.