A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
x64: add APX NDD lowering for simple integer binary ops (#14930)
Johnnie Birch9ddb1597e266
diff --git a/cranelift/assembler-x64/meta/src/instructions/and.rs b/cranelift/assembler-x64/meta/src/instructions/and.rsindex b22e7aa2fb..7806d31284 100644--- a/cranelift/assembler-x64/meta/src/instructions/and.rs+++ b/cranelift/assembler-x64/meta/src/instructions/and.rs@@ -1,5 +1,5 @@-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};#[rustfmt::skip] // Keeps instructions on a single line.pub fn list() -> Vec<Inst> {@@ -29,6 +29,8 @@ pub fn list() -> Vec<Inst> {inst("andw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x23]).r(), _64b | compat),inst("andl", fmt("RM", [rw(r32), r(rm32)]), rex(0x23).r(), _64b | compat),inst("andq", fmt("RM", [rw(r64), r(rm64)]), rex(0x23).w().r(), _64b),+ // APX: NDD form of `AND`, promoted into EVEX map 4.+ inst("andq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x21).r(), _64b & apx),// BMI1 andninst("andnl", fmt("RVM", [w(r32a), r(r32b), r(rm32)]), vex(LZ)._0f38().w0().op(0xF2), (_64b | compat) & bmi1),inst("andnq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), vex(LZ)._0f38().w1().op(0xF2), _64b & bmi1),diff --git a/cranelift/assembler-x64/meta/src/instructions/or.rs b/cranelift/assembler-x64/meta/src/instructions/or.rsindex a99ba36a98..af56d3274b 100644--- a/cranelift/assembler-x64/meta/src/instructions/or.rs+++ b/cranelift/assembler-x64/meta/src/instructions/or.rs@@ -1,5 +1,5 @@-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};#[rustfmt::skip] // Keeps instructions on a single line.pub fn list() -> Vec<Inst> {@@ -22,6 +22,8 @@ pub fn list() -> Vec<Inst> {inst("orw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x0B]).r(), _64b | compat),inst("orl", fmt("RM", [rw(r32), r(rm32)]), rex(0x0B).r(), _64b | compat),inst("orq", fmt("RM", [rw(r64), r(rm64)]), rex(0x0B).w().r(), _64b),+ // APX: NDD form of `OR`, promoted into EVEX map 4.+ inst("orq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x09).r(), _64b & apx),// `LOCK`-prefixed memory-writing instructions.inst("lock_orb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(1).ib(), _64b | compat).custom(Mnemonic),inst("lock_orw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(1).iw(), _64b | compat).custom(Mnemonic),diff --git a/cranelift/assembler-x64/meta/src/instructions/sub.rs b/cranelift/assembler-x64/meta/src/instructions/sub.rsindex e476f7292a..8244725dc8 100644--- a/cranelift/assembler-x64/meta/src/instructions/sub.rs+++ b/cranelift/assembler-x64/meta/src/instructions/sub.rs@@ -1,5 +1,5 @@-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};#[rustfmt::skip] // Keeps instructions on a single line.pub fn list() -> Vec<Inst> {@@ -22,6 +22,13 @@ pub fn list() -> Vec<Inst> {inst("subw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x2B]).r(), _64b | compat),inst("subl", fmt("RM", [rw(r32), r(rm32)]), rex(0x2B).r(), _64b | compat),inst("subq", fmt("RM", [rw(r64), r(rm64)]), rex(0x2B).w().r(), _64b),+ // APX: NDD (new data destination) form of `SUB`, promoted into EVEX+ // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural+ // destination is the `vvvv`-encoded register, so the written operand is+ // placed in the `V` slot (the middle operand of the `RVM` format) while+ // the two sources occupy ModRM.reg and ModRM.rm. `subq` is 64-bit so+ // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `subl` form.+ inst("subq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x29).r(), _64b & apx),// Subtract with borrow.inst("sbbb", fmt("I", [rw(al), r(imm8)]), rex(0x1C).ib(), _64b | compat),inst("sbbw", fmt("I", [rw(ax), r(imm16)]), rex([0x66, 0x1D]).iw(), _64b | compat),diff --git a/cranelift/assembler-x64/meta/src/instructions/xor.rs b/cranelift/assembler-x64/meta/src/instructions/xor.rsindex 08ac5a9f0f..52b93ddc2c 100644--- a/cranelift/assembler-x64/meta/src/instructions/xor.rs+++ b/cranelift/assembler-x64/meta/src/instructions/xor.rs@@ -1,5 +1,5 @@-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};#[rustfmt::skip] // Keeps instructions on a single line.pub fn list() -> Vec<Inst> {@@ -22,6 +22,8 @@ pub fn list() -> Vec<Inst> {inst("xorw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x33]).r(), _64b | compat),inst("xorl", fmt("RM", [rw(r32), r(rm32)]), rex(0x33).r(), _64b | compat),inst("xorq", fmt("RM", [rw(r64), r(rm64)]), rex(0x33).w().r(), _64b),+ // APX: NDD form of `XOR`, promoted into EVEX map 4.+ inst("xorq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x31).r(), _64b & apx),// `LOCK`-prefixed memory-writing instructions.inst("lock_xorb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(6).ib(), _64b | compat).custom(Mnemonic),inst("lock_xorw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(6).iw(), _64b | compat).custom(Mnemonic),diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rsindex 99166feca7..4d8f3a685e 100644--- a/cranelift/assembler-x64/src/fuzz.rs+++ b/cranelift/assembler-x64/src/fuzz.rs@@ -654,4 +654,122 @@ mod test {let assembled = assemble(&inst.into());assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50");}++ /// Byte-level encoding check for the APX NDD (new-data-destination) form of+ /// `SUB`, promoted into EVEX "map 4" via the extended-EVEX prefix.+ ///+ /// Operand placement is the same as [`apx_addq_rvm_ndd_encoding`]: in `RVM`+ /// format the constructor arguments are `[ModRM.reg source,+ /// vvvv destination, ModRM.rm source]`. For `subq %rax, %rcx, %rdx`+ /// (destination `%rax` = 0 in `vvvv`, source `%rcx` = 1 in ModRM.reg,+ /// source `%rdx` = 2 in ModRM.rm), `W = 1`, `ND = 1`, `NF = 0`, and the+ /// legacy `SUB r/m, reg` opcode byte is `0x29`.+ #[test]+ fn apx_subq_rvm_ndd_encoding() {+ use crate::inst::subq_rvm;++ let inst = subq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1829CA");+ }++ /// Companion to [`apx_subq_rvm_ndd_encoding`] covering a memory operand.+ #[test]+ fn apx_subq_rvm_ndd_disp8_is_unscaled() {+ use crate::inst::subq_rvm;+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};++ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {+ base: FuzzReg::new(7),+ simm32: AmodeOffsetPlusKnownOffset {+ simm32: AmodeOffset::new(0x50),+ offset: None,+ },+ trap: None,+ });+ let inst = subq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418297F50");+ }++ #[test]+ fn apx_andq_rvm_ndd_encoding() {+ use crate::inst::andq_rvm;++ let inst = andq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1821CA");+ }++ #[test]+ fn apx_andq_rvm_ndd_disp8_is_unscaled() {+ use crate::inst::andq_rvm;+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};++ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {+ base: FuzzReg::new(7),+ simm32: AmodeOffsetPlusKnownOffset {+ simm32: AmodeOffset::new(0x50),+ offset: None,+ },+ trap: None,+ });+ let inst = andq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418217F50");+ }++ #[test]+ fn apx_orq_rvm_ndd_encoding() {+ use crate::inst::orq_rvm;++ let inst = orq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1809CA");+ }++ #[test]+ fn apx_orq_rvm_ndd_disp8_is_unscaled() {+ use crate::inst::orq_rvm;+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};++ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {+ base: FuzzReg::new(7),+ simm32: AmodeOffsetPlusKnownOffset {+ simm32: AmodeOffset::new(0x50),+ offset: None,+ },+ trap: None,+ });+ let inst = orq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418097F50");+ }++ #[test]+ fn apx_xorq_rvm_ndd_encoding() {+ use crate::inst::xorq_rvm;++ let inst = xorq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1831CA");+ }++ #[test]+ fn apx_xorq_rvm_ndd_disp8_is_unscaled() {+ use crate::inst::xorq_rvm;+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};++ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {+ base: FuzzReg::new(7),+ simm32: AmodeOffsetPlusKnownOffset {+ simm32: AmodeOffset::new(0x50),+ offset: None,+ },+ trap: None,+ });+ let inst = xorq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);+ let assembled = assemble(&inst.into());+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418317F50");+ }}diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 10c885289e..4fde7e2220 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -306,6 +306,18 @@;; `i64` and smaller.+;; APX: when the `has_apx` feature is enabled, use the NDD (new-data-+;; destination) form of `sub` for 64-bit subtracts. Unlike the legacy two-+;; operand `sub`, the NDD form writes its result to a fresh register encoded in+;; the EVEX `vvvv` field, so the register allocator does not need to insert a+;; move to preserve either input.+;;+;; Note that the NDD form computes `dest = rm - reg`, so the minuend goes in+;; ModRM.rm (the second argument here) and the subtrahend in ModRM.reg.+(rule isub_apx_ndd 1 (lower (isub $I64 x y))+ (if-let true (use_apx))+ (x64_subq_rvm (put_in_gpr y) x))+;; Sub two registers.(rule -3 (lower (isub (fits_in_64 ty) x y))(x64_sub ty x y))@@ -370,6 +382,12 @@;; `{i,b}64` and smaller.+;; APX: when the `has_apx` feature is enabled, use the NDD form of `and` for+;; 64-bit integer values.+(rule band_apx_ndd 3 (lower (band $I64 x y))+ (if-let true (use_apx))+ (x64_andq_rvm x y))+;; And two registers.(rule 0 (lower (band ty x y))(if (ty_int_ref_scalar_64 ty))@@ -499,6 +517,12 @@;; `{i,b}64` and smaller.+;; APX: when the `has_apx` feature is enabled, use the NDD form of `or` for+;; 64-bit integer values.+(rule bor_apx_ndd 3 (lower (bor $I64 x y))+ (if-let true (use_apx))+ (x64_orq_rvm x y))+;; Or two registers.(rule 0 (lower (bor ty x y))(if (ty_int_ref_scalar_64 ty))@@ -572,6 +596,12 @@;; `{i,b}64` and smaller.+;; APX: when the `has_apx` feature is enabled, use the NDD form of `xor` for+;; 64-bit integer values.+(rule bxor_apx_ndd 3 (lower (bxor $I64 x y))+ (if-let true (use_apx))+ (x64_xorq_rvm x y))+;; Xor two registers.(rule 0 (lower (bxor ty x y))(if (ty_int_ref_scalar_64 ty))diff --git a/cranelift/filetests/filetests/isa/x64/apx-arith.clif b/cranelift/filetests/filetests/isa/x64/apx-arith.clifnew file mode 100644index 0000000000..c72630051c--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/apx-arith.clif@@ -0,0 +1,72 @@+test compile precise-output+target x86_64 has_apx++function %and_i64(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ v2 = band v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; andq %rdi, %rsi, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x21, 0xfe, 0x48, 0x89+; .byte 0xec, 0x5d, 0xc3++function %or_i64(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ v2 = bor v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; orq %rdi, %rsi, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x09, 0xfe, 0x48, 0x89+; .byte 0xec, 0x5d, 0xc3++function %xor_i64(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ v2 = bxor v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; xorq %rdi, %rsi, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x31, 0xfe, 0x48, 0x89+; .byte 0xec, 0x5d, 0xc3+diff --git a/cranelift/filetests/filetests/isa/x64/apx-sub.clif b/cranelift/filetests/filetests/isa/x64/apx-sub.clifnew file mode 100644index 0000000000..69ee4c51f6--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/apx-sub.clif@@ -0,0 +1,26 @@+test compile precise-output+target x86_64 has_apx++function %sub_i64(i64, i64) -> i64 {+block0(v0: i64, v1: i64):+ v2 = isub v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; subq %rsi, %rdi, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x29, 0xf7, 0x48, 0x89+; .byte 0xec, 0x5d, 0xc3+