Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

aarch64: lower mixed unsigned-by-signed i8 dot to USDOT (FEAT_I8MM) (#14054)

Damilare Darmie Akinlajae8ac8c27f199
diff --git a/cranelift/codegen/meta/src/isa/arm64.rs b/cranelift/codegen/meta/src/isa/arm64.rs
index 1f5f0de237..968ac6e14f 100644
--- a/cranelift/codegen/meta/src/isa/arm64.rs
+++ b/cranelift/codegen/meta/src/isa/arm64.rs
@@ -32,6 +32,14 @@ pub(crate) fn define() -> TargetIsa {
"",
false,
);
+ settings.add_bool(
+ "has_i8mm",
+ "Has Int8 Matrix Multiplication (FEAT_I8MM) support; enables lowering \
+ the mixed unsigned-by-signed i8 dot product to USDOT instead of a \
+ umull/smull + saddlp widening fallback.",
+ "",
+ false,
+ );
settings.add_bool(
"sign_return_address_all",
"If function return address signing is enabled, then apply it to all \
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 893b2a61a0..6bae128cec 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -1853,6 +1853,8 @@
(Fmls)
;; Signed integer dot product (FEAT_DotProd)
(Sdot)
+ ;; Unsigned-by-signed integer dot product (FEAT_I8MM)
+ (Usdot)
))
;; A Vector miscellaneous operation with two registers.
@@ -2076,6 +2078,10 @@
(decl use_dotprod () Inst)
(extern extractor use_dotprod use_dotprod)
+;; Matches any instruction when the `has_i8mm` (FEAT_I8MM) setting is on.
+(decl use_i8mm () Inst)
+(extern extractor use_i8mm use_i8mm)
+
(spec (use_fp16) (provide (not result)))
(decl pure use_fp16 () bool)
(extern constructor use_fp16 use_fp16)
@@ -3503,6 +3509,13 @@
(rule (sdot acc a b)
(vec_rrr_mod (VecALUModOp.Sdot) acc a b (VectorSize.Size32x4)))
+;; USDOT Vd.4S, Vn.16B, Vm.16B -- like `sdot`, but the first source is read as
+;; unsigned bytes and the second as signed. `acc` is both source and
+;; destination; `a` (unsigned) / `b` (signed) are i8x16.
+(decl usdot (Reg Reg Reg) Reg)
+(rule (usdot acc a b)
+ (vec_rrr_mod (VecALUModOp.Usdot) acc a b (VectorSize.Size32x4)))
+
;; Helper for generating a `udf` instruction.
(decl udf (TrapCode) SideEffectNoResult)
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 0c261c267e..b10e412763 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -2769,6 +2769,9 @@ impl MachInstEmit for Inst {
// so it is baked into top11; only Q (from `size`) is variable.
// top11 (Q=0) | q<<9 with bit15_10 yields 0x4E809400 for .4S/.16B.
VecALUModOp::Sdot => (0b000_01110_10_0, 0b100101),
+ // USDOT Vd.4S, Vn.16B, Vm.16B (FEAT_I8MM). Same shape as
+ // SDOT; only the opcode field differs.
+ VecALUModOp::Usdot => (0b000_01110_10_0, 0b100111),
};
sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index acaeb8a431..81ddc33554 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -2290,6 +2290,7 @@ impl Inst {
// Note: the real operand arrangement is .4s, .16b, .16b;
// this debug print renders all lanes as .4s.
VecALUModOp::Sdot => ("sdot", VectorSize::Size32x4),
+ VecALUModOp::Usdot => ("usdot", VectorSize::Size32x4),
};
let rd = pretty_print_vreg_vector(rd.to_reg(), size);
let ri = pretty_print_vreg_vector(ri, size);
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 4b4dec7524..bf364b7340 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -412,6 +412,23 @@
c))))
(sdot c a b))
+;; With FEAT_I8MM, fold the same tree in its mixed unsigned-by-signed form
+;; into a single `usdot`. This is the shape quantized inference produces —
+;; unsigned activations against signed weights — and it differs from the
+;; `sdot` case above only in that one operand widens unsigned.
+(rule 9 (lower (and (use_i8mm)
+ (has_type $I32X4
+ (iadd _
+ (iadd_pairwise _
+ (iadd_pairwise _
+ (swiden_low _ lo @ (imul _ (uwiden_low _ a) (swiden_low _ b)))
+ (swiden_high _ lo))
+ (iadd_pairwise _
+ (swiden_low _ hi @ (imul _ (uwiden_high _ a) (swiden_high _ b)))
+ (swiden_high _ hi)))
+ c))))
+ (usdot c a b))
+
;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule -1 (lower (iabs ty @ (multi_lane _ _) x))
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index dde8371da3..d50940b2ee 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -187,6 +187,14 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
}
}
+ fn use_i8mm(&mut self, _: Inst) -> Option<()> {
+ if self.backend.isa_flags.has_i8mm() {
+ Some(())
+ } else {
+ None
+ }
+ }
+
fn use_fp16(&mut self) -> bool {
self.backend.isa_flags.has_fp16()
}
diff --git a/cranelift/filetests/filetests/isa/aarch64/usdot.clif b/cranelift/filetests/filetests/isa/aarch64/usdot.clif
new file mode 100644
index 0000000000..c9dba6769d
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/usdot.clif
@@ -0,0 +1,35 @@
+test compile precise-output
+set unwind_info=false
+target aarch64 has_i8mm
+
+;; The mixed unsigned-by-signed i8 dot tree folds into a single `usdot`:
+;; `a` widens unsigned, `b` widens signed.
+function %usdot_i8x16(i32x4, i8x16, i8x16) -> i32x4 {
+block0(v0: i32x4, v1: i8x16, v2: i8x16):
+ v3 = uwiden_low v1
+ v4 = swiden_low v2
+ v5 = imul v3, v4
+ v6 = swiden_low v5
+ v7 = swiden_high v5
+ v8 = iadd_pairwise v6, v7
+ v9 = uwiden_high v1
+ v10 = swiden_high v2
+ v11 = imul v9, v10
+ v12 = swiden_low v11
+ v13 = swiden_high v11
+ v14 = iadd_pairwise v12, v13
+ v15 = iadd_pairwise v8, v14
+ v16 = iadd v15, v0
+ return v16
+}
+
+; VCode:
+; block0:
+; usdot v0.4s, v0.4s, v1.4s, v2.4s
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; usdot v0.4s, v1.16b, v2.16b
+; ret
+
diff --git a/cranelift/filetests/filetests/runtests/usdot.clif b/cranelift/filetests/filetests/runtests/usdot.clif
new file mode 100644
index 0000000000..a1060234c0
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/usdot.clif
@@ -0,0 +1,50 @@
+; Execution coverage for the mixed unsigned-by-signed i8 dot product.
+; The lowering only changes aarch64 codegen; running on the interpreter and
+; every native target confirms the result is unchanged either way.
+;
+; The lanes are summed to a scalar before returning: a vector return does
+; not fit in registers on every target, and the per-lane result is already
+; pinned by the precise-output test in isa/aarch64.
+;
+; riscv64 is not listed: its vector backend has no lowering for the lane
+; extraction this reduction needs (and no vector return either), both
+; unrelated to the dot itself. The interpreter plus the three remaining
+; targets cover the semantics.
+test interpret
+test run
+target aarch64
+target aarch64 has_i8mm
+target x86_64
+target s390x
+
+function %usdot_mixed(i32x4, i8x16, i8x16) -> i32 {
+block0(v0: i32x4, v1: i8x16, v2: i8x16):
+ v3 = uwiden_low v1
+ v4 = swiden_low v2
+ v5 = imul v3, v4
+ v6 = swiden_low v5
+ v7 = swiden_high v5
+ v8 = iadd_pairwise v6, v7
+ v9 = uwiden_high v1
+ v10 = swiden_high v2
+ v11 = imul v9, v10
+ v12 = swiden_low v11
+ v13 = swiden_high v11
+ v14 = iadd_pairwise v12, v13
+ v15 = iadd_pairwise v8, v14
+ v16 = iadd v15, v0
+ v17 = extractlane v16, 0
+ v18 = extractlane v16, 1
+ v19 = extractlane v16, 2
+ v20 = extractlane v16, 3
+ v21 = iadd v17, v18
+ v22 = iadd v19, v20
+ v23 = iadd v21, v22
+ return v23
+}
+; 200 read as u8 is 200, not -56 — a signed reading would give a different
+; total, so this pins the unsigned widening.
+; lanes: 4*(200*-1) + 4*(1*3) + 4*(0*5) + 4*(2*-2) = -800 + 12 + 0 - 16
+; run: %usdot_mixed([0 0 0 0], [200 200 200 200 1 1 1 1 0 0 0 0 2 2 2 2], [-1 -1 -1 -1 3 3 3 3 5 5 5 5 -2 -2 -2 -2]) == -804
+; acc [1 2 3 4] plus 4 per lane = [5 6 7 8]
+; run: %usdot_mixed([1 2 3 4], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1]) == 26
diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rs
index bffce9ab73..bfcdbf3088 100644
--- a/cranelift/native/src/lib.rs
+++ b/cranelift/native/src/lib.rs
@@ -126,6 +126,10 @@ pub fn infer_native_flags(isa_builder: &mut dyn Configurable) -> Result<(), &'st
isa_builder.enable("has_dotprod").unwrap();
}
+ if std::arch::is_aarch64_feature_detected!("i8mm") {
+ isa_builder.enable("has_i8mm").unwrap();
+ }
+
if cfg!(target_os = "macos") {
// Pointer authentication is always available on Apple Silicon.
isa_builder.enable("sign_return_address").unwrap();
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 1ceb91a4ac..6534da5b5e 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -4681,6 +4681,7 @@ fn detect_host_feature(feature: &str) -> Option<bool> {
"paca" => Some(std::arch::is_aarch64_feature_detected!("paca")),
"fp16" => Some(std::arch::is_aarch64_feature_detected!("fp16")),
"dotprod" => Some(std::arch::is_aarch64_feature_detected!("dotprod")),
+ "i8mm" => Some(std::arch::is_aarch64_feature_detected!("i8mm")),
_ => None,
};
diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs
index 7c8acbf735..87b7cb4351 100644
--- a/crates/wasmtime/src/engine.rs
+++ b/crates/wasmtime/src/engine.rs
@@ -571,6 +571,7 @@ information about this check\
"has_pauth" => "paca",
"has_fp16" => "fp16",
"has_dotprod" => "dotprod",
+ "has_i8mm" => "i8mm",
// aarch64 features which don't need detection
// No effect on its own.