Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

x64: add APX NDD lowering for simple integer binary ops (#14930)

Johnnie Birch9ddb1597e266
diff --git a/cranelift/assembler-x64/meta/src/instructions/and.rs b/cranelift/assembler-x64/meta/src/instructions/and.rs
index b22e7aa2fb..7806d31284 100644
--- a/cranelift/assembler-x64/meta/src/instructions/and.rs
+++ b/cranelift/assembler-x64/meta/src/instructions/and.rs
@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
#[rustfmt::skip] // Keeps instructions on a single line.
pub fn list() -> Vec<Inst> {
@@ -29,6 +29,8 @@ pub fn list() -> Vec<Inst> {
inst("andw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x23]).r(), _64b | compat),
inst("andl", fmt("RM", [rw(r32), r(rm32)]), rex(0x23).r(), _64b | compat),
inst("andq", fmt("RM", [rw(r64), r(rm64)]), rex(0x23).w().r(), _64b),
+ // APX: NDD form of `AND`, promoted into EVEX map 4.
+ inst("andq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x21).r(), _64b & apx),
// BMI1 andn
inst("andnl", fmt("RVM", [w(r32a), r(r32b), r(rm32)]), vex(LZ)._0f38().w0().op(0xF2), (_64b | compat) & bmi1),
inst("andnq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), vex(LZ)._0f38().w1().op(0xF2), _64b & bmi1),
diff --git a/cranelift/assembler-x64/meta/src/instructions/or.rs b/cranelift/assembler-x64/meta/src/instructions/or.rs
index a99ba36a98..af56d3274b 100644
--- a/cranelift/assembler-x64/meta/src/instructions/or.rs
+++ b/cranelift/assembler-x64/meta/src/instructions/or.rs
@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
#[rustfmt::skip] // Keeps instructions on a single line.
pub fn list() -> Vec<Inst> {
@@ -22,6 +22,8 @@ pub fn list() -> Vec<Inst> {
inst("orw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x0B]).r(), _64b | compat),
inst("orl", fmt("RM", [rw(r32), r(rm32)]), rex(0x0B).r(), _64b | compat),
inst("orq", fmt("RM", [rw(r64), r(rm64)]), rex(0x0B).w().r(), _64b),
+ // APX: NDD form of `OR`, promoted into EVEX map 4.
+ inst("orq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x09).r(), _64b & apx),
// `LOCK`-prefixed memory-writing instructions.
inst("lock_orb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(1).ib(), _64b | compat).custom(Mnemonic),
inst("lock_orw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(1).iw(), _64b | compat).custom(Mnemonic),
diff --git a/cranelift/assembler-x64/meta/src/instructions/sub.rs b/cranelift/assembler-x64/meta/src/instructions/sub.rs
index e476f7292a..8244725dc8 100644
--- a/cranelift/assembler-x64/meta/src/instructions/sub.rs
+++ b/cranelift/assembler-x64/meta/src/instructions/sub.rs
@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
#[rustfmt::skip] // Keeps instructions on a single line.
pub fn list() -> Vec<Inst> {
@@ -22,6 +22,13 @@ pub fn list() -> Vec<Inst> {
inst("subw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x2B]).r(), _64b | compat),
inst("subl", fmt("RM", [rw(r32), r(rm32)]), rex(0x2B).r(), _64b | compat),
inst("subq", fmt("RM", [rw(r64), r(rm64)]), rex(0x2B).w().r(), _64b),
+ // APX: NDD (new data destination) form of `SUB`, promoted into EVEX
+ // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural
+ // destination is the `vvvv`-encoded register, so the written operand is
+ // placed in the `V` slot (the middle operand of the `RVM` format) while
+ // the two sources occupy ModRM.reg and ModRM.rm. `subq` is 64-bit so
+ // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `subl` form.
+ inst("subq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x29).r(), _64b & apx),
// Subtract with borrow.
inst("sbbb", fmt("I", [rw(al), r(imm8)]), rex(0x1C).ib(), _64b | compat),
inst("sbbw", fmt("I", [rw(ax), r(imm16)]), rex([0x66, 0x1D]).iw(), _64b | compat),
diff --git a/cranelift/assembler-x64/meta/src/instructions/xor.rs b/cranelift/assembler-x64/meta/src/instructions/xor.rs
index 08ac5a9f0f..52b93ddc2c 100644
--- a/cranelift/assembler-x64/meta/src/instructions/xor.rs
+++ b/cranelift/assembler-x64/meta/src/instructions/xor.rs
@@ -1,5 +1,5 @@
-use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
+use crate::dsl::{Customization::*, Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{align, evex, fmt, inst, r, rex, rw, sxl, sxq, vex, w};
#[rustfmt::skip] // Keeps instructions on a single line.
pub fn list() -> Vec<Inst> {
@@ -22,6 +22,8 @@ pub fn list() -> Vec<Inst> {
inst("xorw", fmt("RM", [rw(r16), r(rm16)]), rex([0x66, 0x33]).r(), _64b | compat),
inst("xorl", fmt("RM", [rw(r32), r(rm32)]), rex(0x33).r(), _64b | compat),
inst("xorq", fmt("RM", [rw(r64), r(rm64)]), rex(0x33).w().r(), _64b),
+ // APX: NDD form of `XOR`, promoted into EVEX map 4.
+ inst("xorq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x31).r(), _64b & apx),
// `LOCK`-prefixed memory-writing instructions.
inst("lock_xorb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(6).ib(), _64b | compat).custom(Mnemonic),
inst("lock_xorw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(6).iw(), _64b | compat).custom(Mnemonic),
diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs
index 99166feca7..4d8f3a685e 100644
--- a/cranelift/assembler-x64/src/fuzz.rs
+++ b/cranelift/assembler-x64/src/fuzz.rs
@@ -654,4 +654,122 @@ mod test {
let assembled = assemble(&inst.into());
assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50");
}
+
+ /// Byte-level encoding check for the APX NDD (new-data-destination) form of
+ /// `SUB`, promoted into EVEX "map 4" via the extended-EVEX prefix.
+ ///
+ /// Operand placement is the same as [`apx_addq_rvm_ndd_encoding`]: in `RVM`
+ /// format the constructor arguments are `[ModRM.reg source,
+ /// vvvv destination, ModRM.rm source]`. For `subq %rax, %rcx, %rdx`
+ /// (destination `%rax` = 0 in `vvvv`, source `%rcx` = 1 in ModRM.reg,
+ /// source `%rdx` = 2 in ModRM.rm), `W = 1`, `ND = 1`, `NF = 0`, and the
+ /// legacy `SUB r/m, reg` opcode byte is `0x29`.
+ #[test]
+ fn apx_subq_rvm_ndd_encoding() {
+ use crate::inst::subq_rvm;
+
+ let inst = subq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1829CA");
+ }
+
+ /// Companion to [`apx_subq_rvm_ndd_encoding`] covering a memory operand.
+ #[test]
+ fn apx_subq_rvm_ndd_disp8_is_unscaled() {
+ use crate::inst::subq_rvm;
+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+ base: FuzzReg::new(7),
+ simm32: AmodeOffsetPlusKnownOffset {
+ simm32: AmodeOffset::new(0x50),
+ offset: None,
+ },
+ trap: None,
+ });
+ let inst = subq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418297F50");
+ }
+
+ #[test]
+ fn apx_andq_rvm_ndd_encoding() {
+ use crate::inst::andq_rvm;
+
+ let inst = andq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1821CA");
+ }
+
+ #[test]
+ fn apx_andq_rvm_ndd_disp8_is_unscaled() {
+ use crate::inst::andq_rvm;
+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+ base: FuzzReg::new(7),
+ simm32: AmodeOffsetPlusKnownOffset {
+ simm32: AmodeOffset::new(0x50),
+ offset: None,
+ },
+ trap: None,
+ });
+ let inst = andq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418217F50");
+ }
+
+ #[test]
+ fn apx_orq_rvm_ndd_encoding() {
+ use crate::inst::orq_rvm;
+
+ let inst = orq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1809CA");
+ }
+
+ #[test]
+ fn apx_orq_rvm_ndd_disp8_is_unscaled() {
+ use crate::inst::orq_rvm;
+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+ base: FuzzReg::new(7),
+ simm32: AmodeOffsetPlusKnownOffset {
+ simm32: AmodeOffset::new(0x50),
+ offset: None,
+ },
+ trap: None,
+ });
+ let inst = orq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418097F50");
+ }
+
+ #[test]
+ fn apx_xorq_rvm_ndd_encoding() {
+ use crate::inst::xorq_rvm;
+
+ let inst = xorq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1831CA");
+ }
+
+ #[test]
+ fn apx_xorq_rvm_ndd_disp8_is_unscaled() {
+ use crate::inst::xorq_rvm;
+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+ base: FuzzReg::new(7),
+ simm32: AmodeOffsetPlusKnownOffset {
+ simm32: AmodeOffset::new(0x50),
+ offset: None,
+ },
+ trap: None,
+ });
+ let inst = xorq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418317F50");
+ }
}
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 10c885289e..4fde7e2220 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -306,6 +306,18 @@
;; `i64` and smaller.
+;; APX: when the `has_apx` feature is enabled, use the NDD (new-data-
+;; destination) form of `sub` for 64-bit subtracts. Unlike the legacy two-
+;; operand `sub`, the NDD form writes its result to a fresh register encoded in
+;; the EVEX `vvvv` field, so the register allocator does not need to insert a
+;; move to preserve either input.
+;;
+;; Note that the NDD form computes `dest = rm - reg`, so the minuend goes in
+;; ModRM.rm (the second argument here) and the subtrahend in ModRM.reg.
+(rule isub_apx_ndd 1 (lower (isub $I64 x y))
+ (if-let true (use_apx))
+ (x64_subq_rvm (put_in_gpr y) x))
+
;; Sub two registers.
(rule -3 (lower (isub (fits_in_64 ty) x y))
(x64_sub ty x y))
@@ -370,6 +382,12 @@
;; `{i,b}64` and smaller.
+;; APX: when the `has_apx` feature is enabled, use the NDD form of `and` for
+;; 64-bit integer values.
+(rule band_apx_ndd 3 (lower (band $I64 x y))
+ (if-let true (use_apx))
+ (x64_andq_rvm x y))
+
;; And two registers.
(rule 0 (lower (band ty x y))
(if (ty_int_ref_scalar_64 ty))
@@ -499,6 +517,12 @@
;; `{i,b}64` and smaller.
+;; APX: when the `has_apx` feature is enabled, use the NDD form of `or` for
+;; 64-bit integer values.
+(rule bor_apx_ndd 3 (lower (bor $I64 x y))
+ (if-let true (use_apx))
+ (x64_orq_rvm x y))
+
;; Or two registers.
(rule 0 (lower (bor ty x y))
(if (ty_int_ref_scalar_64 ty))
@@ -572,6 +596,12 @@
;; `{i,b}64` and smaller.
+;; APX: when the `has_apx` feature is enabled, use the NDD form of `xor` for
+;; 64-bit integer values.
+(rule bxor_apx_ndd 3 (lower (bxor $I64 x y))
+ (if-let true (use_apx))
+ (x64_xorq_rvm x y))
+
;; Xor two registers.
(rule 0 (lower (bxor ty x y))
(if (ty_int_ref_scalar_64 ty))
diff --git a/cranelift/filetests/filetests/isa/x64/apx-arith.clif b/cranelift/filetests/filetests/isa/x64/apx-arith.clif
new file mode 100644
index 0000000000..c72630051c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/apx-arith.clif
@@ -0,0 +1,72 @@
+test compile precise-output
+target x86_64 has_apx
+
+function %and_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2 = band v0, v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; andq %rdi, %rsi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x21, 0xfe, 0x48, 0x89
+; .byte 0xec, 0x5d, 0xc3
+
+function %or_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2 = bor v0, v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; orq %rdi, %rsi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x09, 0xfe, 0x48, 0x89
+; .byte 0xec, 0x5d, 0xc3
+
+function %xor_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2 = bxor v0, v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; xorq %rdi, %rsi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x31, 0xfe, 0x48, 0x89
+; .byte 0xec, 0x5d, 0xc3
+
diff --git a/cranelift/filetests/filetests/isa/x64/apx-sub.clif b/cranelift/filetests/filetests/isa/x64/apx-sub.clif
new file mode 100644
index 0000000000..69ee4c51f6
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/apx-sub.clif
@@ -0,0 +1,26 @@
+test compile precise-output
+target x86_64 has_apx
+
+function %sub_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2 = isub v0, v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; subq %rsi, %rdi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x29, 0xf7, 0x48, 0x89
+; .byte 0xec, 0x5d, 0xc3
+