Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift(x64): add the EVEX AVX512-VNNI vpdpbusd form (#14006)

Damilare Darmie Akinlajac99bc2de3e72
diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rs
index 5ba888aceb..d7cfdbe5eb 100644
--- a/cranelift/assembler-x64/meta/src/dsl/features.rs
+++ b/cranelift/assembler-x64/meta/src/dsl/features.rs
@@ -97,6 +97,7 @@ pub enum Feature {
cmpxchg16b,
fma,
avx_vnni,
+ avx512vnni,
}
/// List all CPU features.
@@ -129,6 +130,7 @@ pub const ALL_FEATURES: &[Feature] = &[
Feature::cmpxchg16b,
Feature::fma,
Feature::avx_vnni,
+ Feature::avx512vnni,
];
impl fmt::Display for Feature {
diff --git a/cranelift/assembler-x64/meta/src/instructions/pma.rs b/cranelift/assembler-x64/meta/src/instructions/pma.rs
index 4f52f66264..63daf674ae 100644
--- a/cranelift/assembler-x64/meta/src/instructions/pma.rs
+++ b/cranelift/assembler-x64/meta/src/instructions/pma.rs
@@ -1,5 +1,5 @@
-use crate::dsl::{Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, vex, w};
+use crate::dsl::{Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, vex, w};
#[rustfmt::skip] // Keeps instructions on a single line.
pub fn list() -> Vec<Inst> {
@@ -29,5 +29,7 @@ pub fn list() -> Vec<Inst> {
// products summed into a 32-bit lane and accumulated into the first
// (read-write) operand. One op for the pmaddubsw + pmaddwd + paddd chain.
inst("vpdpbusd", fmt("A", [rw(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().w0().op(0x50).r(), (_64b | compat) & avx_vnni),
+ // EVEX AVX512-VNNI form (128-bit), for hosts with AVX512VNNI + AVX512VL.
+ inst("vpdpbusd", fmt("B", [rw(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f38().w0().op(0x50).r(), (_64b | compat) & avx512vnni & avx512vl),
]
}
diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rs
index 9ac426ec1c..d72be07de2 100644
--- a/cranelift/codegen/meta/src/isa/x86.rs
+++ b/cranelift/codegen/meta/src/isa/x86.rs
@@ -89,6 +89,12 @@ pub(crate) fn define() -> TargetIsa {
"AVX512F: CPUID.07H:EBX.AVX512F[bit 16]",
false,
);
+ let has_avx512vnni = settings.add_bool(
+ "has_avx512vnni",
+ "Has support for AVX512VNNI.",
+ "AVX512VNNI: CPUID.07H:ECX.AVX512_VNNI[bit 11]",
+ false,
+ );
let has_popcnt = settings.add_bool(
"has_popcnt",
"Has support for POPCNT.",
@@ -267,7 +273,7 @@ pub(crate) fn define() -> TargetIsa {
let cascadelake = settings.add_preset(
"cascadelake",
"Cascade Lake microarchitecture.",
- preset!(skylake_avx512),
+ preset!(skylake_avx512 && has_avx512vnni),
);
settings.add_preset(
"cooperlake",
@@ -282,7 +288,7 @@ pub(crate) fn define() -> TargetIsa {
let icelake_client = settings.add_preset(
"icelake-client",
"Ice Lake microarchitecture.",
- preset!(cannonlake && has_avx512bitalg),
+ preset!(cannonlake && has_avx512bitalg && has_avx512vnni),
);
// LLVM doesn't use the name "icelake" but Cranelift did in the past; alias it
settings.add_preset(
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index bb08fc2ab2..7f611a7b9b 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -1314,6 +1314,9 @@
(decl pure use_avx_vnni () bool)
(extern constructor use_avx_vnni use_avx_vnni)
+(decl pure use_avx512vnni () bool)
+(extern constructor use_avx512vnni use_avx512vnni)
+
(decl pure has_sse3 () bool)
(extern constructor has_sse3 has_sse3)
@@ -2953,6 +2956,11 @@
(decl x64_vpdpbusd (Xmm Xmm XmmMem128) Xmm)
(rule (x64_vpdpbusd acc uns sgn) (x64_vpdpbusd_a acc uns sgn))
+;; EVEX AVX512-VNNI form of the same op, for hosts with AVX512VNNI+AVX512VL but
+;; not the VEX AVX-VNNI (e.g. Ice Lake, Cascade Lake).
+(decl x64_vpdpbusd_evex (Xmm Xmm XmmMem128) Xmm)
+(rule (x64_vpdpbusd_evex acc uns sgn) (x64_vpdpbusd_b acc uns sgn))
+
;; Helper for creating `insertps` instructions.
(decl x64_insertps (Xmm XmmMem32 u8) Xmm)
(rule 0 (x64_insertps src1 src2 lane) (x64_insertps_a_or_avx src1 src2 lane))
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index a931c0d2f1..49c0be9390 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1593,6 +1593,10 @@ impl asm::AvailableFeatures for &EmitInfo {
fn avx512vbmi(&self) -> bool {
self.isa_flags.has_avx512vbmi()
}
+
+ fn avx512vnni(&self) -> bool {
+ self.isa_flags.has_avx512vnni()
+ }
}
impl MachInstEmit for Inst {
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index d2ef6f5ec9..9a1b00a9a3 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -4065,6 +4065,22 @@
(if-let true (use_avx_vnni))
(x64_vpdpbusd c b a))
+;; Same fold via EVEX AVX512-VNNI, for hosts with AVX512VNNI but not the VEX
+;; AVX-VNNI. Priority 7 keeps the shorter VEX form (rule 8) preferred when both
+;; are available.
+(rule 7 (lower (has_type $I32X4
+ (iadd _
+ (iadd_pairwise _
+ (iadd_pairwise _
+ (swiden_low _ lo @ (imul _ (swiden_low _ a) (uwiden_low _ b)))
+ (swiden_high _ lo))
+ (iadd_pairwise _
+ (swiden_low _ hi @ (imul _ (swiden_high _ a) (uwiden_high _ b)))
+ (swiden_high _ hi)))
+ c)))
+ (if-let true (use_avx512vnni))
+ (x64_vpdpbusd_evex c b a))
+
;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; With SSE4.1 use the `pmovsx*` instructions for this. The `pmovsx*`
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 6e3bf7825a..8c04e7f893 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -521,6 +521,10 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
self.backend.x64_flags.has_avx() && self.backend.x64_flags.has_avx_vnni()
}
+ fn use_avx512vnni(&mut self) -> bool {
+ self.backend.x64_flags.has_avx512vl() && self.backend.x64_flags.has_avx512vnni()
+ }
+
#[inline]
fn has_sse3(&mut self) -> bool {
self.backend.x64_flags.has_sse3()
diff --git a/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd-evex.clif b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd-evex.clif
new file mode 100644
index 0000000000..87b93c270d
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd-evex.clif
@@ -0,0 +1,27 @@
+test compile
+set unwind_info=false
+target x86_64 has_avx has_avx512f has_avx512vl has_avx512vnni
+
+;; With AVX512-VNNI but no VEX AVX-VNNI, the signed*unsigned i8 widening
+;; dot-accumulate tree folds to the EVEX-encoded `vpdpbusd` (rule 7). The VEX
+;; rule (8) cannot fire here because `has_avx_vnni` is not set.
+function %vpdpbusd_evex_i8x16(i8x16, i8x16, i32x4) -> i32x4 {
+block0(v0: i8x16, v1: i8x16, v2: i32x4):
+ v3 = swiden_low v0
+ v4 = uwiden_low v1
+ v5 = imul v3, v4
+ v6 = swiden_high v0
+ v7 = uwiden_high v1
+ v8 = imul v6, v7
+ v9 = swiden_low v5
+ v10 = swiden_high v5
+ v11 = iadd_pairwise v9, v10
+ v12 = swiden_low v8
+ v13 = swiden_high v8
+ v14 = iadd_pairwise v12, v13
+ v15 = iadd_pairwise v11, v14
+ v16 = iadd v15, v2
+ return v16
+}
+
+; check: vpdpbusd
diff --git a/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif
index ee62e8ea7c..79bed40e56 100644
--- a/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif
+++ b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif
@@ -1,6 +1,7 @@
test interpret
test run
target x86_64 has_avx has_avx_vnni
+target x86_64 has_avx has_avx512f has_avx512vl has_avx512vnni
target aarch64
;; Signed*unsigned i8 widening dot-accumulate — the form x64 AVX-VNNI
diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rs
index 0cd1dd1aee..bffce9ab73 100644
--- a/cranelift/native/src/lib.rs
+++ b/cranelift/native/src/lib.rs
@@ -79,6 +79,9 @@ pub fn infer_native_flags(isa_builder: &mut dyn Configurable) -> Result<(), &'st
if std::is_x86_feature_detected!("avxvnni") {
isa_builder.enable("has_avx_vnni").unwrap();
}
+ if std::is_x86_feature_detected!("avx512vnni") {
+ isa_builder.enable("has_avx512vnni").unwrap();
+ }
if std::is_x86_feature_detected!("bmi1") {
isa_builder.enable("has_bmi1").unwrap();
}
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 50d710e567..517ef1e8fc 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -4742,6 +4742,7 @@ fn detect_host_feature(feature: &str) -> Option<bool> {
"avx512f" => Some(std::is_x86_feature_detected!("avx512f")),
"avx512vl" => Some(std::is_x86_feature_detected!("avx512vl")),
"avx512vbmi" => Some(std::is_x86_feature_detected!("avx512vbmi")),
+ "avx512vnni" => Some(std::is_x86_feature_detected!("avx512vnni")),
"lzcnt" => Some(std::is_x86_feature_detected!("lzcnt")),
_ => None,
diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs
index d438a636fa..7c8acbf735 100644
--- a/crates/wasmtime/src/engine.rs
+++ b/crates/wasmtime/src/engine.rs
@@ -609,6 +609,7 @@ information about this check\
"has_avx512f" => "avx512f",
"has_avx512vl" => "avx512vl",
"has_avx512vbmi" => "avx512vbmi",
+ "has_avx512vnni" => "avx512vnni",
"has_lzcnt" => "lzcnt",
// pulley features