A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift(x64): add the EVEX AVX512-VNNI vpdpbusd form (#14006)
Damilare Darmie Akinlajac99bc2de3e72
diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rsindex 5ba888aceb..d7cfdbe5eb 100644--- a/cranelift/assembler-x64/meta/src/dsl/features.rs+++ b/cranelift/assembler-x64/meta/src/dsl/features.rs@@ -97,6 +97,7 @@ pub enum Feature {cmpxchg16b,fma,avx_vnni,+ avx512vnni,}/// List all CPU features.@@ -129,6 +130,7 @@ pub const ALL_FEATURES: &[Feature] = &[Feature::cmpxchg16b,Feature::fma,Feature::avx_vnni,+ Feature::avx512vnni,];impl fmt::Display for Feature {diff --git a/cranelift/assembler-x64/meta/src/instructions/pma.rs b/cranelift/assembler-x64/meta/src/instructions/pma.rsindex 4f52f66264..63daf674ae 100644--- a/cranelift/assembler-x64/meta/src/instructions/pma.rs+++ b/cranelift/assembler-x64/meta/src/instructions/pma.rs@@ -1,5 +1,5 @@-use crate::dsl::{Feature::*, Inst, Length::*, Location::*};-use crate::dsl::{align, fmt, inst, r, rex, rw, vex, w};+use crate::dsl::{Feature::*, Inst, Length::*, Location::*, TupleType::*};+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, vex, w};#[rustfmt::skip] // Keeps instructions on a single line.pub fn list() -> Vec<Inst> {@@ -29,5 +29,7 @@ pub fn list() -> Vec<Inst> {// products summed into a 32-bit lane and accumulated into the first// (read-write) operand. One op for the pmaddubsw + pmaddwd + paddd chain.inst("vpdpbusd", fmt("A", [rw(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().w0().op(0x50).r(), (_64b | compat) & avx_vnni),+ // EVEX AVX512-VNNI form (128-bit), for hosts with AVX512VNNI + AVX512VL.+ inst("vpdpbusd", fmt("B", [rw(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f38().w0().op(0x50).r(), (_64b | compat) & avx512vnni & avx512vl),]}diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rsindex 9ac426ec1c..d72be07de2 100644--- a/cranelift/codegen/meta/src/isa/x86.rs+++ b/cranelift/codegen/meta/src/isa/x86.rs@@ -89,6 +89,12 @@ pub(crate) fn define() -> TargetIsa {"AVX512F: CPUID.07H:EBX.AVX512F[bit 16]",false,);+ let has_avx512vnni = settings.add_bool(+ "has_avx512vnni",+ "Has support for AVX512VNNI.",+ "AVX512VNNI: CPUID.07H:ECX.AVX512_VNNI[bit 11]",+ false,+ );let has_popcnt = settings.add_bool("has_popcnt","Has support for POPCNT.",@@ -267,7 +273,7 @@ pub(crate) fn define() -> TargetIsa {let cascadelake = settings.add_preset("cascadelake","Cascade Lake microarchitecture.",- preset!(skylake_avx512),+ preset!(skylake_avx512 && has_avx512vnni),);settings.add_preset("cooperlake",@@ -282,7 +288,7 @@ pub(crate) fn define() -> TargetIsa {let icelake_client = settings.add_preset("icelake-client","Ice Lake microarchitecture.",- preset!(cannonlake && has_avx512bitalg),+ preset!(cannonlake && has_avx512bitalg && has_avx512vnni),);// LLVM doesn't use the name "icelake" but Cranelift did in the past; alias itsettings.add_preset(diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex bb08fc2ab2..7f611a7b9b 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -1314,6 +1314,9 @@(decl pure use_avx_vnni () bool)(extern constructor use_avx_vnni use_avx_vnni)+(decl pure use_avx512vnni () bool)+(extern constructor use_avx512vnni use_avx512vnni)+(decl pure has_sse3 () bool)(extern constructor has_sse3 has_sse3)@@ -2953,6 +2956,11 @@(decl x64_vpdpbusd (Xmm Xmm XmmMem128) Xmm)(rule (x64_vpdpbusd acc uns sgn) (x64_vpdpbusd_a acc uns sgn))+;; EVEX AVX512-VNNI form of the same op, for hosts with AVX512VNNI+AVX512VL but+;; not the VEX AVX-VNNI (e.g. Ice Lake, Cascade Lake).+(decl x64_vpdpbusd_evex (Xmm Xmm XmmMem128) Xmm)+(rule (x64_vpdpbusd_evex acc uns sgn) (x64_vpdpbusd_b acc uns sgn))+;; Helper for creating `insertps` instructions.(decl x64_insertps (Xmm XmmMem32 u8) Xmm)(rule 0 (x64_insertps src1 src2 lane) (x64_insertps_a_or_avx src1 src2 lane))diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex a931c0d2f1..49c0be9390 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -1593,6 +1593,10 @@ impl asm::AvailableFeatures for &EmitInfo {fn avx512vbmi(&self) -> bool {self.isa_flags.has_avx512vbmi()}++ fn avx512vnni(&self) -> bool {+ self.isa_flags.has_avx512vnni()+ }}impl MachInstEmit for Inst {diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex d2ef6f5ec9..9a1b00a9a3 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -4065,6 +4065,22 @@(if-let true (use_avx_vnni))(x64_vpdpbusd c b a))+;; Same fold via EVEX AVX512-VNNI, for hosts with AVX512VNNI but not the VEX+;; AVX-VNNI. Priority 7 keeps the shorter VEX form (rule 8) preferred when both+;; are available.+(rule 7 (lower (has_type $I32X4+ (iadd _+ (iadd_pairwise _+ (iadd_pairwise _+ (swiden_low _ lo @ (imul _ (swiden_low _ a) (uwiden_low _ b)))+ (swiden_high _ lo))+ (iadd_pairwise _+ (swiden_low _ hi @ (imul _ (swiden_high _ a) (uwiden_high _ b)))+ (swiden_high _ hi)))+ c)))+ (if-let true (use_avx512vnni))+ (x64_vpdpbusd_evex c b a))+;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; With SSE4.1 use the `pmovsx*` instructions for this. The `pmovsx*`diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rsindex 6e3bf7825a..8c04e7f893 100644--- a/cranelift/codegen/src/isa/x64/lower/isle.rs+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs@@ -521,6 +521,10 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {self.backend.x64_flags.has_avx() && self.backend.x64_flags.has_avx_vnni()}+ fn use_avx512vnni(&mut self) -> bool {+ self.backend.x64_flags.has_avx512vl() && self.backend.x64_flags.has_avx512vnni()+ }+#[inline]fn has_sse3(&mut self) -> bool {self.backend.x64_flags.has_sse3()diff --git a/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd-evex.clif b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd-evex.clifnew file mode 100644index 0000000000..87b93c270d--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd-evex.clif@@ -0,0 +1,27 @@+test compile+set unwind_info=false+target x86_64 has_avx has_avx512f has_avx512vl has_avx512vnni++;; With AVX512-VNNI but no VEX AVX-VNNI, the signed*unsigned i8 widening+;; dot-accumulate tree folds to the EVEX-encoded `vpdpbusd` (rule 7). The VEX+;; rule (8) cannot fire here because `has_avx_vnni` is not set.+function %vpdpbusd_evex_i8x16(i8x16, i8x16, i32x4) -> i32x4 {+block0(v0: i8x16, v1: i8x16, v2: i32x4):+ v3 = swiden_low v0+ v4 = uwiden_low v1+ v5 = imul v3, v4+ v6 = swiden_high v0+ v7 = uwiden_high v1+ v8 = imul v6, v7+ v9 = swiden_low v5+ v10 = swiden_high v5+ v11 = iadd_pairwise v9, v10+ v12 = swiden_low v8+ v13 = swiden_high v8+ v14 = iadd_pairwise v12, v13+ v15 = iadd_pairwise v11, v14+ v16 = iadd v15, v2+ return v16+}++; check: vpdpbusddiff --git a/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clifindex ee62e8ea7c..79bed40e56 100644--- a/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif+++ b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif@@ -1,6 +1,7 @@test interprettest runtarget x86_64 has_avx has_avx_vnni+target x86_64 has_avx has_avx512f has_avx512vl has_avx512vnnitarget aarch64;; Signed*unsigned i8 widening dot-accumulate — the form x64 AVX-VNNIdiff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rsindex 0cd1dd1aee..bffce9ab73 100644--- a/cranelift/native/src/lib.rs+++ b/cranelift/native/src/lib.rs@@ -79,6 +79,9 @@ pub fn infer_native_flags(isa_builder: &mut dyn Configurable) -> Result<(), &'stif std::is_x86_feature_detected!("avxvnni") {isa_builder.enable("has_avx_vnni").unwrap();}+ if std::is_x86_feature_detected!("avx512vnni") {+ isa_builder.enable("has_avx512vnni").unwrap();+ }if std::is_x86_feature_detected!("bmi1") {isa_builder.enable("has_bmi1").unwrap();}diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex 50d710e567..517ef1e8fc 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -4742,6 +4742,7 @@ fn detect_host_feature(feature: &str) -> Option<bool> {"avx512f" => Some(std::is_x86_feature_detected!("avx512f")),"avx512vl" => Some(std::is_x86_feature_detected!("avx512vl")),"avx512vbmi" => Some(std::is_x86_feature_detected!("avx512vbmi")),+ "avx512vnni" => Some(std::is_x86_feature_detected!("avx512vnni")),"lzcnt" => Some(std::is_x86_feature_detected!("lzcnt")),_ => None,diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rsindex d438a636fa..7c8acbf735 100644--- a/crates/wasmtime/src/engine.rs+++ b/crates/wasmtime/src/engine.rs@@ -609,6 +609,7 @@ information about this check\"has_avx512f" => "avx512f","has_avx512vl" => "avx512vl","has_avx512vbmi" => "avx512vbmi",+ "has_avx512vnni" => "avx512vnni","has_lzcnt" => "lzcnt",// pulley features