Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift(x64): fold the u8*i8 dot-accumulate tree to vpdpbusd (AVX-VNNI) (#13964)

Damilare Darmie Akinlajad40f4b95baa2
diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rs
index 87e4172b38..5ba888aceb 100644
--- a/cranelift/assembler-x64/meta/src/dsl/features.rs
+++ b/cranelift/assembler-x64/meta/src/dsl/features.rs
@@ -96,6 +96,7 @@ pub enum Feature {
avx512vbmi,
cmpxchg16b,
fma,
+ avx_vnni,
}
/// List all CPU features.
@@ -127,6 +128,7 @@ pub const ALL_FEATURES: &[Feature] = &[
Feature::avx512vbmi,
Feature::cmpxchg16b,
Feature::fma,
+ Feature::avx_vnni,
];
impl fmt::Display for Feature {
diff --git a/cranelift/assembler-x64/meta/src/instructions/pma.rs b/cranelift/assembler-x64/meta/src/instructions/pma.rs
index 1bd25d0ccf..4f52f66264 100644
--- a/cranelift/assembler-x64/meta/src/instructions/pma.rs
+++ b/cranelift/assembler-x64/meta/src/instructions/pma.rs
@@ -24,5 +24,10 @@ pub fn list() -> Vec<Inst> {
// and the saturated result is packed to the destination operand."
inst("pmaddubsw", fmt("A", [rw(xmm1), r(align(xmm_m128))]), rex([0x66, 0x0F, 0x38, 0x04]), (_64b | compat) & ssse3).alt(avx, "vpmaddubsw_b"),
inst("vpmaddubsw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x04), (_64b | compat) & avx),
+ // AVX-VNNI fused byte dot-accumulate: unsigned bytes of the second
+ // operand times signed bytes of the third, each group of four 16-bit
+ // products summed into a 32-bit lane and accumulated into the first
+ // (read-write) operand. One op for the pmaddubsw + pmaddwd + paddd chain.
+ inst("vpdpbusd", fmt("A", [rw(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().w0().op(0x50).r(), (_64b | compat) & avx_vnni),
]
}
diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs
index 290d888f5a..adb0fd1921 100644
--- a/cranelift/assembler-x64/src/fuzz.rs
+++ b/cranelift/assembler-x64/src/fuzz.rs
@@ -9,8 +9,8 @@ use std::vec::Vec;
use std::{format, println};
use crate::{
- AmodeOffset, AmodeOffsetPlusKnownOffset, AsReg, CodeSink, DeferredTarget, Fixed, Gpr, Inst,
- KnownOffset, NonRspGpr, Registers, TrapCode, Xmm,
+ AmodeOffset, AmodeOffsetPlusKnownOffset, AsReg, CodeSink, DeferredTarget, Feature, Features,
+ Fixed, Gpr, Inst, KnownOffset, NonRspGpr, Registers, TrapCode, Xmm,
};
use arbitrary::{Arbitrary, Result, Unstructured};
use capstone::{Capstone, arch::BuildsCapstone, arch::BuildsCapstoneSyntax, arch::x86};
@@ -24,6 +24,13 @@ use capstone::{Capstone, arch::BuildsCapstone, arch::BuildsCapstoneSyntax, arch:
/// fuzzer infrastructure. It may fail during assembly, disassembly, or when
/// comparing the disassembled strings.
pub fn roundtrip(inst: &Inst<FuzzRegs>) {
+ // The bundled capstone build does not disassemble AVX-VNNI instructions, so
+ // the roundtrip oracle has no reference to compare against; skip them. Their
+ // encodings are covered by dedicated filetests.
+ if features_mention(inst.features(), Feature::avx_vnni) {
+ return;
+ }
+
// Check that we can actually assemble this instruction.
let assembled = assemble(inst);
let expected = disassemble(&assembled, inst);
@@ -42,6 +49,17 @@ pub fn roundtrip(inst: &Inst<FuzzRegs>) {
}
}
+/// Whether an instruction's feature term references `target`; used to skip
+/// instructions the disassembler oracle cannot handle.
+fn features_mention(features: &Features, target: Feature) -> bool {
+ match features {
+ Features::And(a, b) | Features::Or(a, b) => {
+ features_mention(a, target) || features_mention(b, target)
+ }
+ Features::Feature(f) => *f == target,
+ }
+}
+
/// Use this assembler to emit machine code into a byte buffer.
///
/// This will skip any traps or label registrations, but this is fine for the
diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rs
index b757e17403..9ac426ec1c 100644
--- a/cranelift/codegen/meta/src/isa/x86.rs
+++ b/cranelift/codegen/meta/src/isa/x86.rs
@@ -53,6 +53,12 @@ pub(crate) fn define() -> TargetIsa {
"FMA: CPUID.01H:ECX.FMA[bit 12]",
false,
);
+ let has_avx_vnni = settings.add_bool(
+ "has_avx_vnni",
+ "Has support for AVX-VNNI.",
+ "AVX-VNNI: CPUID.07H.01H:EAX.AVX_VNNI[bit 4]",
+ false,
+ );
let has_avx512bitalg = settings.add_bool(
"has_avx512bitalg",
"Has support for AVX512BITALG.",
@@ -175,7 +181,7 @@ pub(crate) fn define() -> TargetIsa {
let alderlake = settings.add_preset(
"alderlake",
"Alderlake microarchitecture.",
- preset!(tremont && has_bmi1 && has_bmi2 && has_lzcnt && has_fma),
+ preset!(tremont && has_bmi1 && has_bmi2 && has_lzcnt && has_fma && has_avx_vnni),
);
let sierra_forest = settings.add_preset(
"sierraforest",
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index e1104520d6..bb08fc2ab2 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -1311,6 +1311,9 @@
(decl pure use_fma () bool)
(extern constructor use_fma use_fma)
+(decl pure use_avx_vnni () bool)
+(extern constructor use_avx_vnni use_avx_vnni)
+
(decl pure has_sse3 () bool)
(extern constructor has_sse3 has_sse3)
@@ -2945,6 +2948,11 @@
(decl x64_pmaddubsw (Xmm XmmMem128) Xmm)
(rule (x64_pmaddubsw src1 src2) (x64_pmaddubsw_a_or_avx src1 src2))
+;; AVX-VNNI `vpdpbusd acc, uns, sgn`: acc += dot(u8 lanes of `uns`, i8 lanes of
+;; `sgn`) grouped 16->4. `acc` is the read-write accumulator.
+(decl x64_vpdpbusd (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vpdpbusd acc uns sgn) (x64_vpdpbusd_a acc uns sgn))
+
;; Helper for creating `insertps` instructions.
(decl x64_insertps (Xmm XmmMem32 u8) Xmm)
(rule 0 (x64_insertps src1 src2 lane) (x64_insertps_a_or_avx src1 src2 lane))
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index bc4f249718..a931c0d2f1 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1578,6 +1578,9 @@ impl asm::AvailableFeatures for &EmitInfo {
fn fma(&self) -> bool {
self.isa_flags.has_fma()
}
+ fn avx_vnni(&self) -> bool {
+ self.isa_flags.has_avx_vnni()
+ }
fn avx512dq(&self) -> bool {
self.isa_flags.has_avx512dq()
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 8a3c511c26..d2ef6f5ec9 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -4047,6 +4047,24 @@
(imul _ (swiden_high _ x) (swiden_high _ y))))
(x64_pmaddwd x y))
+;; AVX-VNNI: fold the u8*i8 widening dot-accumulate tree into one `vpdpbusd`
+;; (`a` sign-extended, `b` zero-extended — the signed*unsigned form the
+;; instruction computes). Same tree as aarch64 `sdot` but with `uwiden` on `b`;
+;; priority stays above the scalar `iadd` rules the overlap checker can't prove
+;; disjoint.
+(rule 8 (lower (has_type $I32X4
+ (iadd _
+ (iadd_pairwise _
+ (iadd_pairwise _
+ (swiden_low _ lo @ (imul _ (swiden_low _ a) (uwiden_low _ b)))
+ (swiden_high _ lo))
+ (iadd_pairwise _
+ (swiden_low _ hi @ (imul _ (swiden_high _ a) (uwiden_high _ b)))
+ (swiden_high _ hi)))
+ c)))
+ (if-let true (use_avx_vnni))
+ (x64_vpdpbusd c b a))
+
;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; With SSE4.1 use the `pmovsx*` instructions for this. The `pmovsx*`
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index fb464b2012..6e3bf7825a 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -517,6 +517,10 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
self.backend.x64_flags.has_avx() && self.backend.x64_flags.has_fma()
}
+ fn use_avx_vnni(&mut self) -> bool {
+ self.backend.x64_flags.has_avx() && self.backend.x64_flags.has_avx_vnni()
+ }
+
#[inline]
fn has_sse3(&mut self) -> bool {
self.backend.x64_flags.has_sse3()
diff --git a/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd.clif b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd.clif
new file mode 100644
index 0000000000..9ee197b2c0
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd.clif
@@ -0,0 +1,26 @@
+test compile
+set unwind_info=false
+target x86_64 has_avx has_avx_vnni
+
+;; The signed*unsigned i8 widening dot-accumulate tree (`a` sign-extended, `b`
+;; zero-extended) contracts to one AVX-VNNI `vpdpbusd`.
+function %vpdpbusd_i8x16(i8x16, i8x16, i32x4) -> i32x4 {
+block0(v0: i8x16, v1: i8x16, v2: i32x4):
+ v3 = swiden_low v0
+ v4 = uwiden_low v1
+ v5 = imul v3, v4
+ v6 = swiden_high v0
+ v7 = uwiden_high v1
+ v8 = imul v6, v7
+ v9 = swiden_low v5
+ v10 = swiden_high v5
+ v11 = iadd_pairwise v9, v10
+ v12 = swiden_low v8
+ v13 = swiden_high v8
+ v14 = iadd_pairwise v12, v13
+ v15 = iadd_pairwise v11, v14
+ v16 = iadd v15, v2
+ return v16
+}
+
+; check: vpdpbusd
diff --git a/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif
new file mode 100644
index 0000000000..ee62e8ea7c
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif
@@ -0,0 +1,32 @@
+test interpret
+test run
+target x86_64 has_avx has_avx_vnni
+target aarch64
+
+;; Signed*unsigned i8 widening dot-accumulate — the form x64 AVX-VNNI
+;; `vpdpbusd` computes: `a` sign-extended, `b` zero-extended. On a host with
+;; AVX-VNNI this lowers to a single `vpdpbusd`; elsewhere to the widening chain.
+;; The second case uses b bytes > 127 so the unsigned reading differs from
+;; signed (0xc8 = -56 signed = 200 unsigned).
+function %vpdpbusd_i8x16(i8x16, i8x16, i32x4) -> i32x4 {
+block0(v0: i8x16, v1: i8x16, v2: i32x4):
+ v3 = swiden_low v0
+ v4 = uwiden_low v1
+ v5 = imul v3, v4
+ v6 = swiden_high v0
+ v7 = uwiden_high v1
+ v8 = imul v6, v7
+ v9 = swiden_low v5
+ v10 = swiden_high v5
+ v11 = iadd_pairwise v9, v10
+ v12 = swiden_low v8
+ v13 = swiden_high v8
+ v14 = iadd_pairwise v12, v13
+ v15 = iadd_pairwise v11, v14
+ v16 = iadd v15, v2
+ return v16
+}
+
+; run: %vpdpbusd_i8x16([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [0 0 0 0]) == [10 26 42 58]
+; run: %vpdpbusd_i8x16([1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [-56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56], [0 0 0 0]) == [800 800 800 800]
+; run: %vpdpbusd_i8x16([-1 -2 -3 -4 5 6 7 8 1 1 1 1 2 2 2 2], [10 10 10 10 1 1 1 1 -56 -56 -56 -56 3 3 3 3], [100 0 0 0]) == [0 26 800 24]
diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rs
index 4469df829b..0cd1dd1aee 100644
--- a/cranelift/native/src/lib.rs
+++ b/cranelift/native/src/lib.rs
@@ -76,6 +76,9 @@ pub fn infer_native_flags(isa_builder: &mut dyn Configurable) -> Result<(), &'st
if std::is_x86_feature_detected!("fma") {
isa_builder.enable("has_fma").unwrap();
}
+ if std::is_x86_feature_detected!("avxvnni") {
+ isa_builder.enable("has_avx_vnni").unwrap();
+ }
if std::is_x86_feature_detected!("bmi1") {
isa_builder.enable("has_bmi1").unwrap();
}
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index ebf93ccdec..50d710e567 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -4734,6 +4734,7 @@ fn detect_host_feature(feature: &str) -> Option<bool> {
"avx" => Some(std::is_x86_feature_detected!("avx")),
"avx2" => Some(std::is_x86_feature_detected!("avx2")),
"fma" => Some(std::is_x86_feature_detected!("fma")),
+ "avxvnni" => Some(std::is_x86_feature_detected!("avxvnni")),
"bmi1" => Some(std::is_x86_feature_detected!("bmi1")),
"bmi2" => Some(std::is_x86_feature_detected!("bmi2")),
"avx512bitalg" => Some(std::is_x86_feature_detected!("avx512bitalg")),
diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs
index a3e1bf7fae..d438a636fa 100644
--- a/crates/wasmtime/src/engine.rs
+++ b/crates/wasmtime/src/engine.rs
@@ -601,6 +601,7 @@ information about this check\
"has_avx" => "avx",
"has_avx2" => "avx2",
"has_fma" => "fma",
+ "has_avx_vnni" => "avxvnni",
"has_bmi1" => "bmi1",
"has_bmi2" => "bmi2",
"has_avx512bitalg" => "avx512bitalg",