A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift(x64): fold the u8*i8 dot-accumulate tree to vpdpbusd (AVX-VNNI) (#13964)
Damilare Darmie Akinlajad40f4b95baa2
diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rsindex 87e4172b38..5ba888aceb 100644--- a/cranelift/assembler-x64/meta/src/dsl/features.rs+++ b/cranelift/assembler-x64/meta/src/dsl/features.rs@@ -96,6 +96,7 @@ pub enum Feature {avx512vbmi,cmpxchg16b,fma,+ avx_vnni,}/// List all CPU features.@@ -127,6 +128,7 @@ pub const ALL_FEATURES: &[Feature] = &[Feature::avx512vbmi,Feature::cmpxchg16b,Feature::fma,+ Feature::avx_vnni,];impl fmt::Display for Feature {diff --git a/cranelift/assembler-x64/meta/src/instructions/pma.rs b/cranelift/assembler-x64/meta/src/instructions/pma.rsindex 1bd25d0ccf..4f52f66264 100644--- a/cranelift/assembler-x64/meta/src/instructions/pma.rs+++ b/cranelift/assembler-x64/meta/src/instructions/pma.rs@@ -24,5 +24,10 @@ pub fn list() -> Vec<Inst> {// and the saturated result is packed to the destination operand."inst("pmaddubsw", fmt("A", [rw(xmm1), r(align(xmm_m128))]), rex([0x66, 0x0F, 0x38, 0x04]), (_64b | compat) & ssse3).alt(avx, "vpmaddubsw_b"),inst("vpmaddubsw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x04), (_64b | compat) & avx),+ // AVX-VNNI fused byte dot-accumulate: unsigned bytes of the second+ // operand times signed bytes of the third, each group of four 16-bit+ // products summed into a 32-bit lane and accumulated into the first+ // (read-write) operand. One op for the pmaddubsw + pmaddwd + paddd chain.+ inst("vpdpbusd", fmt("A", [rw(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().w0().op(0x50).r(), (_64b | compat) & avx_vnni),]}diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rsindex 290d888f5a..adb0fd1921 100644--- a/cranelift/assembler-x64/src/fuzz.rs+++ b/cranelift/assembler-x64/src/fuzz.rs@@ -9,8 +9,8 @@ use std::vec::Vec;use std::{format, println};use crate::{- AmodeOffset, AmodeOffsetPlusKnownOffset, AsReg, CodeSink, DeferredTarget, Fixed, Gpr, Inst,- KnownOffset, NonRspGpr, Registers, TrapCode, Xmm,+ AmodeOffset, AmodeOffsetPlusKnownOffset, AsReg, CodeSink, DeferredTarget, Feature, Features,+ Fixed, Gpr, Inst, KnownOffset, NonRspGpr, Registers, TrapCode, Xmm,};use arbitrary::{Arbitrary, Result, Unstructured};use capstone::{Capstone, arch::BuildsCapstone, arch::BuildsCapstoneSyntax, arch::x86};@@ -24,6 +24,13 @@ use capstone::{Capstone, arch::BuildsCapstone, arch::BuildsCapstoneSyntax, arch:/// fuzzer infrastructure. It may fail during assembly, disassembly, or when/// comparing the disassembled strings.pub fn roundtrip(inst: &Inst<FuzzRegs>) {+ // The bundled capstone build does not disassemble AVX-VNNI instructions, so+ // the roundtrip oracle has no reference to compare against; skip them. Their+ // encodings are covered by dedicated filetests.+ if features_mention(inst.features(), Feature::avx_vnni) {+ return;+ }+// Check that we can actually assemble this instruction.let assembled = assemble(inst);let expected = disassemble(&assembled, inst);@@ -42,6 +49,17 @@ pub fn roundtrip(inst: &Inst<FuzzRegs>) {}}+/// Whether an instruction's feature term references `target`; used to skip+/// instructions the disassembler oracle cannot handle.+fn features_mention(features: &Features, target: Feature) -> bool {+ match features {+ Features::And(a, b) | Features::Or(a, b) => {+ features_mention(a, target) || features_mention(b, target)+ }+ Features::Feature(f) => *f == target,+ }+}+/// Use this assembler to emit machine code into a byte buffer.////// This will skip any traps or label registrations, but this is fine for thediff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rsindex b757e17403..9ac426ec1c 100644--- a/cranelift/codegen/meta/src/isa/x86.rs+++ b/cranelift/codegen/meta/src/isa/x86.rs@@ -53,6 +53,12 @@ pub(crate) fn define() -> TargetIsa {"FMA: CPUID.01H:ECX.FMA[bit 12]",false,);+ let has_avx_vnni = settings.add_bool(+ "has_avx_vnni",+ "Has support for AVX-VNNI.",+ "AVX-VNNI: CPUID.07H.01H:EAX.AVX_VNNI[bit 4]",+ false,+ );let has_avx512bitalg = settings.add_bool("has_avx512bitalg","Has support for AVX512BITALG.",@@ -175,7 +181,7 @@ pub(crate) fn define() -> TargetIsa {let alderlake = settings.add_preset("alderlake","Alderlake microarchitecture.",- preset!(tremont && has_bmi1 && has_bmi2 && has_lzcnt && has_fma),+ preset!(tremont && has_bmi1 && has_bmi2 && has_lzcnt && has_fma && has_avx_vnni),);let sierra_forest = settings.add_preset("sierraforest",diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex e1104520d6..bb08fc2ab2 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -1311,6 +1311,9 @@(decl pure use_fma () bool)(extern constructor use_fma use_fma)+(decl pure use_avx_vnni () bool)+(extern constructor use_avx_vnni use_avx_vnni)+(decl pure has_sse3 () bool)(extern constructor has_sse3 has_sse3)@@ -2945,6 +2948,11 @@(decl x64_pmaddubsw (Xmm XmmMem128) Xmm)(rule (x64_pmaddubsw src1 src2) (x64_pmaddubsw_a_or_avx src1 src2))+;; AVX-VNNI `vpdpbusd acc, uns, sgn`: acc += dot(u8 lanes of `uns`, i8 lanes of+;; `sgn`) grouped 16->4. `acc` is the read-write accumulator.+(decl x64_vpdpbusd (Xmm Xmm XmmMem128) Xmm)+(rule (x64_vpdpbusd acc uns sgn) (x64_vpdpbusd_a acc uns sgn))+;; Helper for creating `insertps` instructions.(decl x64_insertps (Xmm XmmMem32 u8) Xmm)(rule 0 (x64_insertps src1 src2 lane) (x64_insertps_a_or_avx src1 src2 lane))diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex bc4f249718..a931c0d2f1 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -1578,6 +1578,9 @@ impl asm::AvailableFeatures for &EmitInfo {fn fma(&self) -> bool {self.isa_flags.has_fma()}+ fn avx_vnni(&self) -> bool {+ self.isa_flags.has_avx_vnni()+ }fn avx512dq(&self) -> bool {self.isa_flags.has_avx512dq()diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 8a3c511c26..d2ef6f5ec9 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -4047,6 +4047,24 @@(imul _ (swiden_high _ x) (swiden_high _ y))))(x64_pmaddwd x y))+;; AVX-VNNI: fold the u8*i8 widening dot-accumulate tree into one `vpdpbusd`+;; (`a` sign-extended, `b` zero-extended — the signed*unsigned form the+;; instruction computes). Same tree as aarch64 `sdot` but with `uwiden` on `b`;+;; priority stays above the scalar `iadd` rules the overlap checker can't prove+;; disjoint.+(rule 8 (lower (has_type $I32X4+ (iadd _+ (iadd_pairwise _+ (iadd_pairwise _+ (swiden_low _ lo @ (imul _ (swiden_low _ a) (uwiden_low _ b)))+ (swiden_high _ lo))+ (iadd_pairwise _+ (swiden_low _ hi @ (imul _ (swiden_high _ a) (uwiden_high _ b)))+ (swiden_high _ hi)))+ c)))+ (if-let true (use_avx_vnni))+ (x64_vpdpbusd c b a))+;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; With SSE4.1 use the `pmovsx*` instructions for this. The `pmovsx*`diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rsindex fb464b2012..6e3bf7825a 100644--- a/cranelift/codegen/src/isa/x64/lower/isle.rs+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs@@ -517,6 +517,10 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {self.backend.x64_flags.has_avx() && self.backend.x64_flags.has_fma()}+ fn use_avx_vnni(&mut self) -> bool {+ self.backend.x64_flags.has_avx() && self.backend.x64_flags.has_avx_vnni()+ }+#[inline]fn has_sse3(&mut self) -> bool {self.backend.x64_flags.has_sse3()diff --git a/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd.clif b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd.clifnew file mode 100644index 0000000000..9ee197b2c0--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/simd-vpdpbusd.clif@@ -0,0 +1,26 @@+test compile+set unwind_info=false+target x86_64 has_avx has_avx_vnni++;; The signed*unsigned i8 widening dot-accumulate tree (`a` sign-extended, `b`+;; zero-extended) contracts to one AVX-VNNI `vpdpbusd`.+function %vpdpbusd_i8x16(i8x16, i8x16, i32x4) -> i32x4 {+block0(v0: i8x16, v1: i8x16, v2: i32x4):+ v3 = swiden_low v0+ v4 = uwiden_low v1+ v5 = imul v3, v4+ v6 = swiden_high v0+ v7 = uwiden_high v1+ v8 = imul v6, v7+ v9 = swiden_low v5+ v10 = swiden_high v5+ v11 = iadd_pairwise v9, v10+ v12 = swiden_low v8+ v13 = swiden_high v8+ v14 = iadd_pairwise v12, v13+ v15 = iadd_pairwise v11, v14+ v16 = iadd v15, v2+ return v16+}++; check: vpdpbusddiff --git a/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clifnew file mode 100644index 0000000000..ee62e8ea7c--- /dev/null+++ b/cranelift/filetests/filetests/runtests/simd-vpdpbusd.clif@@ -0,0 +1,32 @@+test interpret+test run+target x86_64 has_avx has_avx_vnni+target aarch64++;; Signed*unsigned i8 widening dot-accumulate — the form x64 AVX-VNNI+;; `vpdpbusd` computes: `a` sign-extended, `b` zero-extended. On a host with+;; AVX-VNNI this lowers to a single `vpdpbusd`; elsewhere to the widening chain.+;; The second case uses b bytes > 127 so the unsigned reading differs from+;; signed (0xc8 = -56 signed = 200 unsigned).+function %vpdpbusd_i8x16(i8x16, i8x16, i32x4) -> i32x4 {+block0(v0: i8x16, v1: i8x16, v2: i32x4):+ v3 = swiden_low v0+ v4 = uwiden_low v1+ v5 = imul v3, v4+ v6 = swiden_high v0+ v7 = uwiden_high v1+ v8 = imul v6, v7+ v9 = swiden_low v5+ v10 = swiden_high v5+ v11 = iadd_pairwise v9, v10+ v12 = swiden_low v8+ v13 = swiden_high v8+ v14 = iadd_pairwise v12, v13+ v15 = iadd_pairwise v11, v14+ v16 = iadd v15, v2+ return v16+}++; run: %vpdpbusd_i8x16([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [0 0 0 0]) == [10 26 42 58]+; run: %vpdpbusd_i8x16([1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [-56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56 -56], [0 0 0 0]) == [800 800 800 800]+; run: %vpdpbusd_i8x16([-1 -2 -3 -4 5 6 7 8 1 1 1 1 2 2 2 2], [10 10 10 10 1 1 1 1 -56 -56 -56 -56 3 3 3 3], [100 0 0 0]) == [0 26 800 24]diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rsindex 4469df829b..0cd1dd1aee 100644--- a/cranelift/native/src/lib.rs+++ b/cranelift/native/src/lib.rs@@ -76,6 +76,9 @@ pub fn infer_native_flags(isa_builder: &mut dyn Configurable) -> Result<(), &'stif std::is_x86_feature_detected!("fma") {isa_builder.enable("has_fma").unwrap();}+ if std::is_x86_feature_detected!("avxvnni") {+ isa_builder.enable("has_avx_vnni").unwrap();+ }if std::is_x86_feature_detected!("bmi1") {isa_builder.enable("has_bmi1").unwrap();}diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex ebf93ccdec..50d710e567 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -4734,6 +4734,7 @@ fn detect_host_feature(feature: &str) -> Option<bool> {"avx" => Some(std::is_x86_feature_detected!("avx")),"avx2" => Some(std::is_x86_feature_detected!("avx2")),"fma" => Some(std::is_x86_feature_detected!("fma")),+ "avxvnni" => Some(std::is_x86_feature_detected!("avxvnni")),"bmi1" => Some(std::is_x86_feature_detected!("bmi1")),"bmi2" => Some(std::is_x86_feature_detected!("bmi2")),"avx512bitalg" => Some(std::is_x86_feature_detected!("avx512bitalg")),diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rsindex a3e1bf7fae..d438a636fa 100644--- a/crates/wasmtime/src/engine.rs+++ b/crates/wasmtime/src/engine.rs@@ -601,6 +601,7 @@ information about this check\"has_avx" => "avx","has_avx2" => "avx2","has_fma" => "fma",+ "has_avx_vnni" => "avxvnni","has_bmi1" => "bmi1","has_bmi2" => "bmi2","has_avx512bitalg" => "avx512bitalg",