Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

x64: add APX NDD form of 64-bit add (#13916)

Johnnie Birch31ddba9ef0cb
diff --git a/cranelift/assembler-x64/meta/src/dsl.rs b/cranelift/assembler-x64/meta/src/dsl.rs
index 69e7778d38..293d914620 100644
--- a/cranelift/assembler-x64/meta/src/dsl.rs
+++ b/cranelift/assembler-x64/meta/src/dsl.rs
@@ -10,8 +10,8 @@ mod features;
pub mod format;
pub use custom::{Custom, Customization};
+pub use encoding::{ApxClass, Evex, Length, Vex, VexEscape, VexPrefix, evex, vex};
pub use encoding::{Encoding, ModRmKind, OpcodeMod};
-pub use encoding::{Evex, Length, Vex, VexEscape, VexPrefix, evex, vex};
pub use encoding::{
Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, TupleType, rex,
};
diff --git a/cranelift/assembler-x64/meta/src/dsl/encoding.rs b/cranelift/assembler-x64/meta/src/dsl/encoding.rs
index c843e609d7..2d254be9c4 100644
--- a/cranelift/assembler-x64/meta/src/dsl/encoding.rs
+++ b/cranelift/assembler-x64/meta/src/dsl/encoding.rs
@@ -57,6 +57,9 @@ pub fn evex(length: Length, tuple_type: TupleType) -> Evex {
modrm: None,
imm: Imm::None,
tuple_type,
+ apx: None,
+ nd: None,
+ nf: None,
}
}
@@ -86,6 +89,13 @@ impl Encoding {
Encoding::Evex(evex) => evex.opcode,
}
}
+
+ /// Return whether this encoding sets the APX `ND` ("new data destination")
+ /// bit, meaning the architectural destination is the `vvvv`-encoded
+ /// register rather than an operand named by ModRM.
+ pub fn is_nd(&self) -> bool {
+ matches!(self, Encoding::Evex(evex) if evex.nd == Some(true))
+ }
}
impl fmt::Display for Encoding {
@@ -841,6 +851,10 @@ pub enum VexEscape {
_0F,
_0F3A,
_0F38,
+ /// APX "opcode map 4"; only valid for APX legacy-GPR (Extended EVEX)
+ /// encodings, never for VEX. This is the map that enables promoting legacy
+ /// general-purpose-register instructions into the EVEX space.
+ _MAP4,
}
impl VexEscape {
@@ -851,6 +865,7 @@ impl VexEscape {
Self::_0F => 0b01,
Self::_0F38 => 0b10,
Self::_0F3A => 0b11,
+ Self::_MAP4 => 0b100,
}
}
}
@@ -861,6 +876,7 @@ impl fmt::Display for VexEscape {
Self::_0F => write!(f, "0F"),
Self::_0F3A => write!(f, "0F3A"),
Self::_0F38 => write!(f, "0F38"),
+ Self::_MAP4 => write!(f, "MAP4"),
}
}
}
@@ -1247,6 +1263,30 @@ pub struct Evex {
/// The "Tuple Type" corresponding to scaling of the 8-bit displacement
/// parameter for memory operands. See [`TupleType`] for more information.
pub tuple_type: TupleType,
+ /// The APX "Extended EVEX" class, when this instruction is an APX
+ /// promotion.
+ ///
+ /// Intel APX (Advanced Performance Extensions) reuses the `0x62` EVEX
+ /// identifier but does *not* define a single static payload layout.
+ /// Instead, the meaning of the payload bytes (`P0`, `P1`, `P2`) is
+ /// re-mapped depending on which class of instruction is being promoted (see
+ /// [`ApxClass`]). `None` denotes a standard (AVX-512) EVEX encoding;
+ /// `Some(_)` selects one of the APX layouts and enables addressing of the
+ /// extended general-purpose registers `R16`–`R31` ("EGPR").
+ pub apx: Option<ApxClass>,
+ /// The APX `ND` (New Data destination) bit.
+ ///
+ /// When set, a legacy destructive two-operand instruction is promoted into
+ /// a non-destructive three-operand form (the extra destination is encoded
+ /// in the `EVEX.vvvv` field). `None` when the bit is not part of this
+ /// encoding; only valid for [`ApxClass::LegacyGpr`].
+ pub nd: Option<bool>,
+ /// The APX `NF` (No Flags) bit.
+ ///
+ /// When set, the status-flag writes that a legacy integer instruction would
+ /// otherwise perform are suppressed. `None` when the bit is not part of
+ /// this encoding; only valid for [`ApxClass::LegacyGpr`].
+ pub nf: Option<bool>,
}
impl Evex {
@@ -1310,6 +1350,27 @@ impl Evex {
}
}
+ /// Select the APX "opcode map 4", promoting a legacy general-purpose-register
+ /// instruction into the Extended EVEX space; equivalent to `.MAP4` in the
+ /// manual.
+ ///
+ /// This both sets the `mmm` map bits and marks the encoding as
+ /// [`ApxClass::LegacyGpr`], which in turn enables addressing of the extended
+ /// GPRs `R16`–`R31` ("EGPR") and permits the `ND`/`NF` bits.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the map (`mmm`) or APX class has already been set.
+ pub fn map4(self) -> Self {
+ assert!(self.mmm.is_none());
+ assert!(self.apx.is_none());
+ Self {
+ mmm: Some(VexEscape::_MAP4),
+ apx: Some(ApxClass::LegacyGpr),
+ ..self
+ }
+ }
+
/// Set the `W` bit to `0`; equivalent to `.W0` in the manual.
pub fn w0(self) -> Self {
assert!(self.w.is_ignored());
@@ -1352,9 +1413,72 @@ impl Evex {
}
}
+ /// Mark this as an APX "Extended EVEX" encoding of the given [`ApxClass`].
+ ///
+ /// This selects the APX payload layout to emit and enables addressing of
+ /// the extended general-purpose registers `R16`–`R31` ("EGPR").
+ ///
+ /// # Panics
+ ///
+ /// Panics if an APX class has already been set.
+ pub fn apx(self, class: ApxClass) -> Self {
+ assert!(self.apx.is_none());
+ Self {
+ apx: Some(class),
+ ..self
+ }
+ }
+
+ /// Set the APX `ND` (New Data destination) bit; equivalent to `.ND` in the
+ /// manual.
+ ///
+ /// # Panics
+ ///
+ /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit
+ /// has already been set.
+ pub fn nd(self) -> Self {
+ assert_eq!(
+ self.apx,
+ Some(ApxClass::LegacyGpr),
+ "the ND bit is only valid for APX legacy-GPR promotions"
+ );
+ assert!(self.nd.is_none());
+ Self {
+ nd: Some(true),
+ ..self
+ }
+ }
+
+ /// Set the APX `NF` (No Flags) bit; equivalent to `.NF` in the manual.
+ ///
+ /// # Panics
+ ///
+ /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit
+ /// has already been set.
+ pub fn nf(self) -> Self {
+ assert_eq!(
+ self.apx,
+ Some(ApxClass::LegacyGpr),
+ "the NF bit is only valid for APX legacy-GPR promotions"
+ );
+ assert!(self.nf.is_none());
+ Self {
+ nf: Some(true),
+ ..self
+ }
+ }
+
fn validate(&self, _operands: &[Operand]) {
assert!(self.opcode != u8::MAX);
assert!(self.mmm.is_some());
+ // The `ND`/`NF` bits are only defined for APX legacy-GPR promotions.
+ if self.nd.is_some() || self.nf.is_some() {
+ assert_eq!(
+ self.apx,
+ Some(ApxClass::LegacyGpr),
+ "the ND/NF bits require an APX legacy-GPR encoding"
+ );
+ }
}
/// Retrieve the digit extending the opcode, if available.
@@ -1412,7 +1536,14 @@ impl fmt::Display for Evex {
if let Some(mmmmm) = self.mmm {
write!(f, ".{mmmmm}")?;
}
- write!(f, ".{} {:#04X}", self.w, self.opcode)?;
+ write!(f, ".{}", self.w)?;
+ if self.nd == Some(true) {
+ write!(f, ".ND")?;
+ }
+ if self.nf == Some(true) {
+ write!(f, ".NF")?;
+ }
+ write!(f, " {:#04X}", self.opcode)?;
if let Some(modrm) = self.modrm {
write!(f, " {modrm}")?;
}
@@ -1423,6 +1554,35 @@ impl fmt::Display for Evex {
}
}
+/// The class of an APX "Extended EVEX" encoding.
+///
+/// Intel APX does not define a single static Extended-EVEX layout. Although
+/// every APX instruction still begins with the `0x62` EVEX identifier, the
+/// meaning of the payload bytes (`P0`, `P1`, `P2`) is re-mapped depending on
+/// the class of instruction being promoted. This enum selects which layout the
+/// assembler should emit; all classes gain access to the extended
+/// general-purpose registers `R16`–`R31` ("EGPR").
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum ApxClass {
+ /// Promotion of a legacy general-purpose-register (GPR) instruction using
+ /// extended opcode map 4. Only this class may set the `ND` and `NF` bits.
+ LegacyGpr,
+ /// Promotion of a legacy SSE / VEX vector instruction into the EVEX space.
+ Vector,
+ /// An existing AVX-512 (EVEX) instruction extended with APX register bits.
+ Avx512,
+}
+
+impl fmt::Display for ApxClass {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ Self::LegacyGpr => write!(f, "LegacyGpr"),
+ Self::Vector => write!(f, "Vector"),
+ Self::Avx512 => write!(f, "Avx512"),
+ }
+ }
+}
+
/// Tuple Type definitions used in EVEX encodings.
///
/// This enumeration corresponds to table 2-34 and 2-35 in the Intel manual.
diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rs
index d7cfdbe5eb..e9266d7db5 100644
--- a/cranelift/assembler-x64/meta/src/dsl/features.rs
+++ b/cranelift/assembler-x64/meta/src/dsl/features.rs
@@ -98,6 +98,7 @@ pub enum Feature {
fma,
avx_vnni,
avx512vnni,
+ apx,
}
/// List all CPU features.
@@ -131,6 +132,7 @@ pub const ALL_FEATURES: &[Feature] = &[
Feature::fma,
Feature::avx_vnni,
Feature::avx512vnni,
+ Feature::apx,
];
impl fmt::Display for Feature {
diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rs
index e1e4a8bb3b..6cb6c1cd5b 100644
--- a/cranelift/assembler-x64/meta/src/generate/format.rs
+++ b/cranelift/assembler-x64/meta/src/generate/format.rs
@@ -54,29 +54,46 @@ impl dsl::Format {
/// once Cranelift has switched to using this assembler predominantly
/// (TODO).
#[must_use]
- pub(crate) fn generate_att_style_operands(&self) -> String {
- let ordered_ops: Vec<_> = self
- .operands
- .iter()
- .filter(|o| !o.implicit)
- .rev()
- .map(|o| format!("{{{}}}", o.location))
- .collect();
- ordered_ops.join(", ")
+ pub(crate) fn generate_att_style_operands(&self, nd: bool) -> String {
+ self.ordered_operands(nd, false)
}
/// Like [`Self::generate_att_style_operands`], but omits the fixed `%xmm0`
/// mask operand, which XED leaves implicit.
#[must_use]
- pub(crate) fn generate_xed_style_operands(&self) -> String {
- let ordered_ops: Vec<_> = self
+ pub(crate) fn generate_xed_style_operands(&self, nd: bool) -> String {
+ self.ordered_operands(nd, true)
+ }
+
+ /// Shared operand ordering for AT&T-style printing.
+ ///
+ /// Normally this is just the reverse of the DSL's Intel-style order. APX
+ /// "new data destination" (NDD) forms are the exception: their
+ /// architectural destination is the `vvvv`-encoded register, which the DSL
+ /// must list in the middle so that the positional slot assignment in
+ /// `generate_vex_or_evex_prefix` maps operands to `reg`/`vvvv`/`rm`
+ /// correctly. Blindly reversing would therefore print the destination in
+ /// the middle. Instead, for `ND = 1` the sources keep their DSL order and
+ /// the destination is printed last, which matches both AT&T convention and
+ /// the XED disassembly used as a fuzzing oracle.
+ #[must_use]
+ fn ordered_operands(&self, nd: bool, xed_style: bool) -> String {
+ let ops = self
.operands
.iter()
- .filter(|o| !o.implicit && o.location != dsl::Location::xmm0)
- .rev()
+ .filter(|o| !o.implicit && !(xed_style && o.location == dsl::Location::xmm0));
+ let ordered: Vec<&dsl::Operand> = if nd {
+ let (dst, srcs): (Vec<_>, Vec<_>) =
+ ops.partition(|o| matches!(o.mutability, dsl::Mutability::Write));
+ srcs.into_iter().chain(dst).collect()
+ } else {
+ ops.rev().collect()
+ };
+ ordered
+ .into_iter()
.map(|o| format!("{{{}}}", o.location))
- .collect();
- ordered_ops.join(", ")
+ .collect::<Vec<_>>()
+ .join(", ")
}
#[must_use]
@@ -243,24 +260,51 @@ impl dsl::Format {
fmtln!(f, "let w = {};", vex.w.as_bool());
let bits = "len, pp, mmmmm, w";
- self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, || {
- vex.unwrap_digit()
- })
+ self.generate_vex_or_evex_prefix(
+ f,
+ "VexPrefix",
+ &bits,
+ vex.is4,
+ None,
+ "two_op",
+ "three_op",
+ || vex.unwrap_digit(),
+ )
}
fn generate_evex_prefix(&self, f: &mut Formatter, evex: &dsl::Evex) -> ModRmStyle {
f.empty_line();
f.comment("Emit EVEX prefix.");
- let ll = evex.length.evex_bits();
- fmtln!(f, "let ll = {ll:#04b};");
+
+ // Intel APX promotes legacy GPR instructions into "EVEX map 4" using a
+ // re-purposed payload layout (see `EvexPrefix::legacy` in the runtime
+ // assembler and section 3.1.2.3.1 of the APX spec). In that case we
+ // emit the ND/NF bits and select the `legacy_*` constructors instead of
+ // the AVX-512 ones.
+ let apx_legacy = matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr));
+
fmtln!(f, "let pp = {:#04b};", evex.pp.map_or(0b00, |pp| pp.bits()));
fmtln!(f, "let mmm = {:#07b};", evex.mmm.unwrap().bits());
fmtln!(f, "let w = {};", evex.w.as_bool());
- // NB: when bcast is supported in the future the `evex_scaling`
- // calculation for `Full` and `Half` below need to be updated.
+
+ let (bits, two_op, three_op);
+ if apx_legacy {
+ fmtln!(f, "let nd = {};", evex.nd == Some(true));
+ fmtln!(f, "let nf = {};", evex.nf == Some(true));
+ bits = String::from("pp, mmm, w, nd, nf");
+ two_op = "legacy_two_op";
+ three_op = "legacy_three_op";
+ } else {
+ let ll = evex.length.evex_bits();
+ fmtln!(f, "let ll = {ll:#04b};");
+ // NB: when bcast is supported in the future the `evex_scaling`
+ // calculation for `Full` and `Half` below need to be updated.
+ fmtln!(f, "let bcast = false;");
+ bits = String::from("ll, pp, mmm, w, bcast");
+ two_op = "two_op";
+ three_op = "three_op";
+ }
let bcast = false;
- fmtln!(f, "let bcast = {bcast};");
- let bits = format!("ll, pp, mmm, w, bcast");
let is4 = false;
let length_bytes = match evex.length {
@@ -273,39 +317,59 @@ impl dsl::Format {
// Figure out, according to table 2-34 and 2-35 in the Intel manual,
// what the scaling factor is for 8-bit displacements to pass through to
// encoding.
- let evex_scaling = Some(match evex.tuple_type {
- dsl::TupleType::Full => {
- assert!(!bcast);
- length_bytes
- }
- dsl::TupleType::Half => {
- assert!(!bcast);
- length_bytes / 2
- }
- dsl::TupleType::FullMem => length_bytes,
- // FIXME: according to table 2-35 this needs to take into account
- // "InputSize" which isn't accounted for in our `Evex` structure at
- // this time.
- dsl::TupleType::Tuple1Scalar => unimplemented!(),
- dsl::TupleType::Tuple1Fixed => unimplemented!(),
- dsl::TupleType::Tuple2 => unimplemented!(),
- dsl::TupleType::Tuple4 => unimplemented!(),
- dsl::TupleType::Tuple8 => 32,
- dsl::TupleType::HalfMem => length_bytes / 2,
- dsl::TupleType::QuarterMem => length_bytes / 4,
- dsl::TupleType::EigthMem => length_bytes / 8,
- dsl::TupleType::Mem128 => 16,
- dsl::TupleType::Movddup => match evex.length {
- dsl::Length::LZ | dsl::Length::LIG => unimplemented!(),
- dsl::Length::L128 => 8,
- dsl::Length::L256 => 32,
- dsl::Length::L512 => 64,
- },
- });
+ //
+ // The compressed-displacement scheme of section 2.7.5 only applies to
+ // the vector EVEX encodings. APX promotes legacy general-purpose-
+ // register instructions into extended-EVEX "map 4", and those keep
+ // legacy displacement semantics: `disp8` is a plain byte offset rather
+ // than a multiple of a tuple-derived scaling factor `N`. Scaling them
+ // would emit an offset wrong by a factor of `N`. Note this is specific
+ // to the legacy-GPR class; promoted vector instructions and APX-extended
+ // AVX-512 instructions still use compressed displacements.
+ let evex_scaling = if matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr)) {
+ None
+ } else {
+ Some(match evex.tuple_type {
+ dsl::TupleType::Full => {
+ assert!(!bcast);
+ length_bytes
+ }
+ dsl::TupleType::Half => {
+ assert!(!bcast);
+ length_bytes / 2
+ }
+ dsl::TupleType::FullMem => length_bytes,
+ // FIXME: according to table 2-35 this needs to take into account
+ // "InputSize" which isn't accounted for in our `Evex` structure at
+ // this time.
+ dsl::TupleType::Tuple1Scalar => unimplemented!(),
+ dsl::TupleType::Tuple1Fixed => unimplemented!(),
+ dsl::TupleType::Tuple2 => unimplemented!(),
+ dsl::TupleType::Tuple4 => unimplemented!(),
+ dsl::TupleType::Tuple8 => 32,
+ dsl::TupleType::HalfMem => length_bytes / 2,
+ dsl::TupleType::QuarterMem => length_bytes / 4,
+ dsl::TupleType::EigthMem => length_bytes / 8,
+ dsl::TupleType::Mem128 => 16,
+ dsl::TupleType::Movddup => match evex.length {
+ dsl::Length::LZ | dsl::Length::LIG => unimplemented!(),
+ dsl::Length::L128 => 8,
+ dsl::Length::L256 => 32,
+ dsl::Length::L512 => 64,
+ },
+ })
+ };
- self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, || {
- evex.unwrap_digit()
- })
+ self.generate_vex_or_evex_prefix(
+ f,
+ "EvexPrefix",
+ &bits,
+ is4,
+ evex_scaling,
+ two_op,
+ three_op,
+ || evex.unwrap_digit(),
+ )
}
/// Helper function to generate either a vex or evex prefix, mostly handling
@@ -318,6 +382,8 @@ impl dsl::Format {
bits: &str,
is4: bool,
evex_scaling: Option<i8>,
+ two_op: &str,
+ three_op: &str,
unwrap_digit: impl Fn() -> Option<u8>,
) -> ModRmStyle {
use dsl::OperandKind::{FixedReg, Imm, Mem, Reg, RegMem};
@@ -330,7 +396,7 @@ impl dsl::Format {
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
fmtln!(
f,
- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
);
ModRmStyle::Reg {
reg: ModRmReg::Reg(*reg),
@@ -347,7 +413,7 @@ impl dsl::Format {
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
fmtln!(
f,
- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
);
ModRmStyle::RegMem {
reg: ModRmReg::Reg(*reg),
@@ -362,7 +428,7 @@ impl dsl::Format {
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
fmtln!(
f,
- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
);
ModRmStyle::RegMemIs4 {
reg: ModRmReg::Reg(*reg),
@@ -382,7 +448,7 @@ impl dsl::Format {
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
fmtln!(
f,
- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
);
ModRmStyle::RegMem {
reg: ModRmReg::Digit(digit),
@@ -395,7 +461,7 @@ impl dsl::Format {
let reg = reg_or_vvvv;
fmtln!(f, "let reg = self.{reg}.enc();");
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
- fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");
+ fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");
ModRmStyle::RegMem {
reg: ModRmReg::Reg(*reg),
rm: *rm,
@@ -413,7 +479,7 @@ impl dsl::Format {
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
fmtln!(
f,
- "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+ "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
);
ModRmStyle::Reg {
reg: ModRmReg::Digit(digit),
@@ -425,7 +491,7 @@ impl dsl::Format {
let reg = reg_or_vvvv;
fmtln!(f, "let reg = self.{reg}.enc();");
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
- fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");
+ fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");
ModRmStyle::Reg {
reg: ModRmReg::Reg(*reg),
rm: *rm,
@@ -437,7 +503,7 @@ impl dsl::Format {
assert!(!is4);
fmtln!(f, "let reg = self.{reg}.enc();");
fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
- fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");
+ fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");
ModRmStyle::RegMem {
reg: ModRmReg::Reg(*reg),
rm: *rm,
diff --git a/cranelift/assembler-x64/meta/src/generate/inst.rs b/cranelift/assembler-x64/meta/src/generate/inst.rs
index a2eb137356..71ec243abd 100644
--- a/cranelift/assembler-x64/meta/src/generate/inst.rs
+++ b/cranelift/assembler-x64/meta/src/generate/inst.rs
@@ -324,8 +324,9 @@ impl dsl::Inst {
None => fmtln!(f, "let {location} = {to_string};"),
}
}
- let ordered_ops = self.format.generate_att_style_operands();
- let xed_ops = self.format.generate_xed_style_operands();
+ let nd = self.encoding.is_nd();
+ let ordered_ops = self.format.generate_att_style_operands(nd);
+ let xed_ops = self.format.generate_xed_style_operands(nd);
let mut implicit_ops = self.format.generate_implicit_operands();
if self.has_trap {
fmtln!(f, "let trap = self.trap;");
diff --git a/cranelift/assembler-x64/meta/src/instructions/add.rs b/cranelift/assembler-x64/meta/src/instructions/add.rs
index f5969061b0..f4b6b4fb29 100644
--- a/cranelift/assembler-x64/meta/src/instructions/add.rs
+++ b/cranelift/assembler-x64/meta/src/instructions/add.rs
@@ -97,5 +97,12 @@ pub fn list() -> Vec<Inst> {
inst("vphaddw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x01).r(), (_64b | compat) & avx),
inst("vphaddd", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x02).r(), (_64b | compat) & avx),
inst("vaddpd", fmt("C", [w(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f().w1().op(0x58).r(), (_64b | compat) & avx512vl),
+ // APX: NDD (new data destination) form of `ADD`, promoted into EVEX
+ // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural
+ // destination is the `vvvv`-encoded register, so the written operand is
+ // placed in the `V` slot (the middle operand of the `RVM` format) while
+ // the two sources occupy ModRM.reg and ModRM.rm. `addq` is 64-bit so
+ // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `addl` form.
+ inst("addq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b & apx),
]
}
diff --git a/cranelift/assembler-x64/src/evex.rs b/cranelift/assembler-x64/src/evex.rs
index bf89dd13bd..2734fae7ac 100644
--- a/cranelift/assembler-x64/src/evex.rs
+++ b/cranelift/assembler-x64/src/evex.rs
@@ -92,6 +92,107 @@ impl EvexPrefix {
EvexPrefix::new(reg, vvvv, (b, x), ll, pp, mmm, w, broadcast)
}
+ // ---------------------------------------------------------------------
+ // Intel APX "Extended EVEX" prefix for promoted *legacy* GPR instructions
+ // (EVEX map 4). See the Intel APX Architecture Specification (rev 8),
+ // section 3.1.2.3.1 "EVEX Extension of Legacy Instructions", Figure 3.3.
+ //
+ // The four bytes still begin with `0x62`, but several payload bits are
+ // re-purposed relative to the AVX-512 layout above:
+ //
+ // ┌────┬────┬────┬────┬────┬────┬────┬────┐
+ // Byte 1: │ R3 │ X3 │ B3 │ R4 │ B4 │ 1 │ 0 │ 0 │ (map id = 4)
+ // ├────┼────┼────┼────┼────┼────┼────┼────┤
+ // Byte 2: │ W │ V3 │ V2 │ V1 │ V0 │ U │ p │ p │ (U = ~X4)
+ // ├────┼────┼────┼────┼────┼────┼────┼────┤
+ // Byte 3: │ 0 │ 0 │ 0 │ ND │ V4 │ NF │ 0 │ 0 │
+ // └────┴────┴────┴────┴────┴────┴────┴────┘
+ //
+ // The "underlined" fields (`R3`, `X3`, `B3`, `R4`, the `vvvv` bits and `V4`)
+ // are stored inverted, exactly as in the AVX-512 layout. `B4` and `X4` are
+ // newly repurposed reserved bits: `B4` uses *true* polarity (fixed value 0)
+ // and `X4` is carried inverted in the `U` bit (`EVEX.X4 = ~EVEX.U`), so a
+ // register-form instruction (ModRM.Mod = 3, no index) has `U = 1`.
+
+ /// Construct the extended-EVEX (APX map 4) prefix for a legacy GPR
+ /// instruction.
+ ///
+ /// `reg` is the ModRM.reg register, `vvvv` is the `V` register identifier
+ /// (the NDD register when `nd` is set), and `(b, x)` are the ModRM.r/m base
+ /// and (optional) SIB index registers. `nd`/`nf` select the New Data
+ /// destination and No Flags bits respectively.
+ pub fn legacy(
+ reg: u8,
+ vvvv: u8,
+ (b, x): (Option<u8>, Option<u8>),
+ pp: u8,
+ mmm: u8,
+ w: bool,
+ nd: bool,
+ nf: bool,
+ ) -> Self {
+ let base = b.unwrap_or(0);
+ let index = x.unwrap_or(0);
+
+ // byte1 (P0)
+ let r3 = invert_top_bit(reg);
+ let x3 = invert_top_bit(index);
+ let b3 = invert_top_bit(base);
+ let r4 = invert_top_bit(reg >> 1);
+ let b4 = (base >> 4) & 1; // true polarity
+ debug_assert!(mmm <= 0b111);
+ let byte1 = r3 << 7 | x3 << 6 | b3 << 5 | r4 << 4 | b4 << 3 | mmm;
+
+ // byte2 (P1)
+ debug_assert!(vvvv <= 0b11111);
+ debug_assert!(pp <= 0b11);
+ let vvvv_value = !vvvv & 0b1111;
+ // `EVEX.X4 = ~EVEX.U`; with no index register X4 = 0 so U = 1.
+ let x4 = (index >> 4) & 1;
+ let u = (!x4) & 1;
+ let byte2 = (w as u8) << 7 | vvvv_value << 3 | u << 2 | (pp & 0b11);
+
+ // byte3 (P2)
+ let v_prime = invert_top_bit(vvvv >> 1); // V4, inverted
+ let byte3 = (nd as u8) << 4 | v_prime << 3 | (nf as u8) << 2;
+
+ Self {
+ byte1,
+ byte2,
+ byte3,
+ }
+ }
+
+ /// Construct the extended-EVEX (APX map 4) prefix for a two-operand legacy
+ /// GPR instruction (no NDD); the `V` register identifier is unused.
+ #[allow(dead_code, reason = "not all APX legacy forms are emitted yet")]
+ pub fn legacy_two_op(
+ reg: u8,
+ (b, x): (Option<u8>, Option<u8>),
+ pp: u8,
+ mmm: u8,
+ w: bool,
+ nd: bool,
+ nf: bool,
+ ) -> Self {
+ EvexPrefix::legacy(reg, 0, (b, x), pp, mmm, w, nd, nf)
+ }
+
+ /// Construct the extended-EVEX (APX map 4) prefix for a three-operand
+ /// legacy GPR instruction; `vvvv` carries the NDD register.
+ pub fn legacy_three_op(
+ reg: u8,
+ vvvv: u8,
+ (b, x): (Option<u8>, Option<u8>),
+ pp: u8,
+ mmm: u8,
+ w: bool,
+ nd: bool,
+ nf: bool,
+ ) -> Self {
+ EvexPrefix::legacy(reg, vvvv, (b, x), pp, mmm, w, nd, nf)
+ }
+
pub(crate) fn encode(&self, sink: &mut impl CodeSink) {
sink.put1(0x62);
sink.put1(self.byte1);
diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs
index 3efe0acfc4..99166feca7 100644
--- a/cranelift/assembler-x64/src/fuzz.rs
+++ b/cranelift/assembler-x64/src/fuzz.rs
@@ -34,6 +34,13 @@ pub fn roundtrip(inst: &Inst<FuzzRegs>) {
return;
}
+ // Likewise, capstone cannot decode the APX extended-EVEX ("map 4")
+ // encodings at all, returning zero instructions. These are instead checked
+ // against XED by `roundtrip_xed`, which does understand map 4.
+ if features_mention(inst.features(), Feature::apx) {
+ return;
+ }
+
roundtrip_with(
inst,
"capstone",
@@ -589,4 +596,62 @@ mod test {
})
.budget_ms(1_000);
}
+
+ /// Byte-level encoding check for the APX NDD (new-data-destination) form of
+ /// `ADD`, promoted into EVEX "map 4" via the extended-EVEX prefix.
+ ///
+ /// We assert the exact bytes rather than round-tripping through Capstone
+ /// because the bundled disassembler does not yet understand APX. With
+ /// `ND = 1` the architectural destination is the `vvvv`-encoded register, so
+ /// the `RVM` operands are `[ModRM.reg source, vvvv destination, ModRM.rm
+ /// source]`. For `addq %rax, %rcx, %rdx` (destination `%rax` = 0 in `vvvv`,
+ /// source `%rcx` = 1 in ModRM.reg, source `%rdx` = 2 in ModRM.rm), `W = 1`,
+ /// `ND = 1`, `NF = 0`, the expected sequence is:
+ ///
+ /// ```text
+ /// 62 F4 FC 18 01 CA
+ /// ^^ ^^ ^^ ^^ ^^ ^^
+ /// | | | | | └ ModRM: mod=11 reg=rcx r/m=rdx
+ /// | | | | └ opcode 0x01 (ADD r/m, reg)
+ /// | | | └ P2: ND=1 (bit4), V4=1 (bit3, inverted), NF=0
+ /// | | └ P1: W=1, vvvv=~rax=1111, U=1, pp=00
+ /// | └ P0: R3 X3 B3 R4 = 1111, B4=0, map=100 (map 4)
+ /// └ EVEX identifier
+ /// ```
+ #[test]
+ fn apx_addq_rvm_ndd_encoding() {
+ use crate::inst::addq_rvm;
+ // Format is `RVM` = [ModRM.reg source, vvvv destination, ModRM.rm
+ // source], so the constructor arguments are (reg source = %rcx,
+ // destination = %rax, r/m source = %rdx).
+ let inst = addq_rvm::<FuzzRegs>::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1801CA");
+ }
+
+ /// Companion to [`apx_addq_rvm_ndd_encoding`] covering a memory operand.
+ ///
+ /// APX map-4 instructions are legacy instructions promoted into EVEX, so
+ /// their `disp8` keeps legacy semantics: a plain byte offset. The
+ /// compressed-displacement scheme that divides `disp8` by a tuple-derived
+ /// factor `N` applies only to the vector EVEX encodings. Encoding
+ /// `0x50(%rdi)` must therefore emit `0x50` and not `0x50 / 16 = 0x05`,
+ /// which would silently address the wrong memory.
+ #[test]
+ fn apx_addq_rvm_ndd_disp8_is_unscaled() {
+ use crate::inst::addq_rvm;
+ use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+ let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+ base: FuzzReg::new(7),
+ simm32: AmodeOffsetPlusKnownOffset {
+ simm32: AmodeOffset::new(0x50),
+ offset: None,
+ },
+ trap: None,
+ });
+ let inst = addq_rvm::<FuzzRegs>::new(FuzzReg::new(7), FuzzReg::new(7), mem);
+ let assembled = assemble(&inst.into());
+ assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50");
+ }
}
diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rs
index d72be07de2..836338027f 100644
--- a/cranelift/codegen/meta/src/isa/x86.rs
+++ b/cranelift/codegen/meta/src/isa/x86.rs
@@ -95,6 +95,15 @@ pub(crate) fn define() -> TargetIsa {
"AVX512VNNI: CPUID.07H:ECX.AVX512_VNNI[bit 11]",
false,
);
+ // Registered so `target x86_64 has_apx` parses; not yet referenced by any
+ // preset because no shipping microarchitecture ships APX today. Drop the
+ // leading underscore once a preset needs it.
+ let _has_apx = settings.add_bool(
+ "has_apx",
+ "Has support for APX.",
+ "APX_F: CPUID.(EAX=07H,ECX=1H):EDX.APX_F[bit 21]",
+ false,
+ );
let has_popcnt = settings.add_bool(
"has_popcnt",
"Has support for POPCNT.",
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 24c1d7ded8..62d14633ac 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -1336,6 +1336,9 @@
(decl pure use_avx2 () bool)
(extern constructor use_avx2 use_avx2)
+(decl pure use_apx () bool)
+(extern constructor use_apx use_apx)
+
(decl pure has_cmpxchg16b () bool)
(extern constructor has_cmpxchg16b has_cmpxchg16b)
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 3f2b6aa4ac..1ee94966b3 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1595,6 +1595,10 @@ impl asm::AvailableFeatures for &EmitInfo {
fn avx512vnni(&self) -> bool {
self.isa_flags.has_avx512vnni()
}
+
+ fn apx(&self) -> bool {
+ self.isa_flags.has_apx()
+ }
}
impl MachInstEmit for Inst {
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index ed9b8fac38..617616445f 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -94,6 +94,15 @@
(rule iadd_base_case_32_or_64_lea -5 (lower (iadd (ty_32_or_64 ty) x y))
(x64_lea ty (to_amode_add (mem_flags_trusted_data) x y (zero_offset))))
+;; APX: when the `has_apx` feature is enabled, use the NDD (new-data-
+;; destination) form of `add` for 64-bit adds. Unlike the legacy two-operand
+;; `add`, the NDD form writes its result to a fresh register encoded in the
+;; EVEX `vvvv` field, so the register allocator does not need to insert a move
+;; to preserve either input.
+(rule iadd_apx_ndd 1 (lower (iadd $I64 x y))
+ (if-let true (use_apx))
+ (x64_addq_rvm x y))
+
;; Higher-priority cases than the previous two where a load can be sunk into
;; the add instruction itself. Note that both operands are tested for
;; sink-ability since addition is commutative
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 8c04e7f893..c5b33e1623 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -492,6 +492,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
self.backend.x64_flags.has_avx512vbmi()
}
+ #[inline]
+ fn use_apx(&mut self) -> bool {
+ self.backend.x64_flags.has_apx()
+ }
+
#[inline]
fn has_lzcnt(&mut self) -> bool {
self.backend.x64_flags.has_lzcnt()
diff --git a/cranelift/filetests/filetests/isa/x64/apx-add.clif b/cranelift/filetests/filetests/isa/x64/apx-add.clif
new file mode 100644
index 0000000000..94fcbfd180
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/apx-add.clif
@@ -0,0 +1,26 @@
+test compile precise-output
+target x86_64 has_apx
+
+function %add_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+ v2 = iadd v0, v1
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; addq %rdi, %rsi, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; .byte 0x62, 0xf4, 0xfc, 0x18, 0x01, 0xfe, 0x48, 0x89
+; .byte 0xec, 0x5d, 0xc3
+